diff options
Diffstat (limited to 'infra/libkookie/nixpkgs/pkgs/development/python-modules/transformers')
-rw-r--r-- | infra/libkookie/nixpkgs/pkgs/development/python-modules/transformers/default.nix | 22 |
1 files changed, 14 insertions, 8 deletions
diff --git a/infra/libkookie/nixpkgs/pkgs/development/python-modules/transformers/default.nix b/infra/libkookie/nixpkgs/pkgs/development/python-modules/transformers/default.nix index fdb807bd0a1e..39fd7a11d0df 100644 --- a/infra/libkookie/nixpkgs/pkgs/development/python-modules/transformers/default.nix +++ b/infra/libkookie/nixpkgs/pkgs/development/python-modules/transformers/default.nix @@ -2,13 +2,14 @@ , stdenv , fetchFromGitHub , boto3 +, cookiecutter , filelock , regex , requests , numpy , parameterized +, protobuf , sacremoses -, sentencepiece , timeout-decorator , tokenizers , tqdm @@ -17,23 +18,23 @@ buildPythonPackage rec { pname = "transformers"; - version = "3.3.1"; + version = "4.0.0"; src = fetchFromGitHub { owner = "huggingface"; repo = pname; rev = "v${version}"; - sha256 = "1j9nzhl0zw5z9rnvzfih7v6bax353rxp05b3f0cvkii3b5dbkc2j"; + sha256 = "17djq32pq8d6vqip7i9pda0ldigmzckbbcd278llmpxdriqd4llg"; }; propagatedBuildInputs = [ - boto3 + cookiecutter filelock numpy + protobuf regex requests sacremoses - sentencepiece tokenizers tqdm ]; @@ -46,7 +47,7 @@ buildPythonPackage rec { postPatch = '' substituteInPlace setup.py \ - --replace "tokenizers == 0.8.1.rc2" "tokenizers>=0.8" + --replace "tokenizers == 0.9.4" "tokenizers" ''; preCheck = '' @@ -67,11 +68,15 @@ buildPythonPackage rec { # Disable tests that require network access. disabledTests = [ - "PegasusTokenizationTest" - "T5TokenizationTest" + "BlenderbotSmallTokenizerTest" + "Blenderbot3BTokenizerTests" + "GetFromCacheTests" + "TokenizationTest" + "TestTokenizationBart" "test_all_tokenizers" "test_batch_encoding_is_fast" "test_batch_encoding_pickle" + "test_batch_encoding_word_to_tokens" "test_config_from_model_shortcut" "test_config_model_type_from_model_identifier" "test_from_pretrained_use_fast_toggle" @@ -86,6 +91,7 @@ buildPythonPackage rec { "test_tokenizer_from_pretrained" "test_tokenizer_from_tokenizer_class" "test_tokenizer_identifier_with_correct_config" + "test_tokenizer_identifier_non_existent" ]; meta = with stdenv.lib; { |