gpt2tc: init at 2020-12-30

This commit is contained in:
Anna Kudriavtsev 2021-03-13 22:51:01 -08:00
parent eb362f9af9
commit fab9608163
4 changed files with 70 additions and 0 deletions

View file

@ -0,0 +1,10 @@
diff --git a/gpt2convert.py b/gpt2convert.py
index 34ca909..6e6cac5 100644
--- a/gpt2convert.py
+++ b/gpt2convert.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
import sys
import tensorflow as tf
import numpy as np

View file

@ -0,0 +1,11 @@
diff --git a/download_model.sh b/download_model.sh
index 9cb401f..ad1dc62 100755
--- a/download_model.sh
+++ b/download_model.sh
@@ -13,5 +13,5 @@ mkdir -p models/$model
for filename in checkpoint encoder.json hparams.json model.ckpt.data-00000-of-00001 model.ckpt.index model.ckpt.meta vocab.bpe; do
fetch=$model/$filename
echo "Fetching $fetch"
- curl --output models/$fetch https://storage.googleapis.com/gpt-2/models/$fetch
+ curl --output models/$fetch https://openaipublic.blob.core.windows.net/gpt-2/models/$fetch
done

View file

@ -0,0 +1,47 @@
{ lib, stdenv, fetchurl, autoPatchelfHook, python3 }:
stdenv.mkDerivation rec {
pname = "gpt2tc";
version = "2020-12-30";
src = fetchurl {
url = "https://bellard.org/nncp/gpt2tc-${version}.tar.gz";
hash = "sha256-KFcl7E6iGx50JaJI1jwzKAdkrkbNngPbrEA/ZSyG+uY=";
};
patches = [
# Add a shebang to the python script so that nix detects it as such and
# wraps it properly. Otherwise, it runs in shell and freezes the system.
./0001-add-python-shebang.patch
# Update the source URL for the models because the old one is down.
./0002-fix-download-url.patch
];
nativeBuildInputs = [ autoPatchelfHook ];
buildInputs = [
(python3.withPackages (p: with p; [ numpy tensorflow ]))
];
dontBuild = true;
installPhase = ''
runHook preInstall
install -D -m755 -t $out/bin gpt2tc
install -T -m755 download_model.sh $out/bin/gpt2-download-model
install -T -m755 gpt2convert.py $out/bin/gpt2-convert
install -D -m644 -t $out/share/gpt2tc readme.txt gpt2vocab.txt Changelog
runHook postInstall
'';
meta = with lib; {
description = "Text completion and compression using GPT-2";
homepage = "https://bellard.org/nncp/gpt2tc.html";
license = licenses.unfree;
platforms = [ "x86_64-linux" ];
maintainers = with maintainers; [ anna328p ];
};
}

View file

@ -4910,6 +4910,8 @@ in
gparted = callPackage ../tools/misc/gparted { };
gpt2tc = callPackage ../tools/text/gpt2tc { };
ldmtool = callPackage ../tools/misc/ldmtool { };
gpodder = callPackage ../applications/audio/gpodder { };