1 files changed, 14 insertions, 8 deletions
diff --git a/infra/libkookie/nixpkgs/pkgs/development/python-modules/transformers/default.nix b/infra/libkookie/nixpkgs/pkgs/development/python-modules/transformers/default.nix
index fdb807bd0a1e..39fd7a11d0df 100644
--- a/infra/libkookie/nixpkgs/pkgs/development/python-modules/transformers/default.nix
+++ b/infra/libkookie/nixpkgs/pkgs/development/python-modules/transformers/default.nix
@@ -2,13 +2,14 @@
 , stdenv
 , fetchFromGitHub
 , boto3
+, cookiecutter
 , filelock
 , regex
 , requests
 , numpy
 , parameterized
+, protobuf
 , sacremoses
-, sentencepiece
 , timeout-decorator
 , tokenizers
 , tqdm
@@ -17,23 +18,23 @@
 
 buildPythonPackage rec {
   pname = "transformers";
-  version = "3.3.1";
+  version = "4.0.0";
 
   src = fetchFromGitHub {
     owner = "huggingface";
     repo = pname;
     rev = "v${version}";
-    sha256 = "1j9nzhl0zw5z9rnvzfih7v6bax353rxp05b3f0cvkii3b5dbkc2j";
+    sha256 = "17djq32pq8d6vqip7i9pda0ldigmzckbbcd278llmpxdriqd4llg";
   };
 
   propagatedBuildInputs = [
-    boto3
+    cookiecutter
     filelock
     numpy
+    protobuf
     regex
     requests
     sacremoses
-    sentencepiece
     tokenizers
     tqdm
   ];
@@ -46,7 +47,7 @@ buildPythonPackage rec {
 
   postPatch = ''
     substituteInPlace setup.py \
-      --replace "tokenizers == 0.8.1.rc2" "tokenizers>=0.8"
+      --replace "tokenizers == 0.9.4" "tokenizers"
   '';
 
   preCheck = ''
@@ -67,11 +68,15 @@ buildPythonPackage rec {
 
   # Disable tests that require network access.
   disabledTests = [
-    "PegasusTokenizationTest"
-    "T5TokenizationTest"
+    "BlenderbotSmallTokenizerTest"
+    "Blenderbot3BTokenizerTests"
+    "GetFromCacheTests"
+    "TokenizationTest"
+    "TestTokenizationBart"
     "test_all_tokenizers"
     "test_batch_encoding_is_fast"
     "test_batch_encoding_pickle"
+    "test_batch_encoding_word_to_tokens"
     "test_config_from_model_shortcut"
     "test_config_model_type_from_model_identifier"
     "test_from_pretrained_use_fast_toggle"
@@ -86,6 +91,7 @@ buildPythonPackage rec {
     "test_tokenizer_from_pretrained"
     "test_tokenizer_from_tokenizer_class"
     "test_tokenizer_identifier_with_correct_config"
+    "test_tokenizer_identifier_non_existent"
   ];
 
   meta = with stdenv.lib; {