1 files changed, 12 insertions, 4 deletions
diff --git a/pkgs/development/python-modules/transformers/default.nix b/pkgs/development/python-modules/transformers/default.nix
index e3fd150fbcf7..39fd7a11d0df 100644
--- a/pkgs/development/python-modules/transformers/default.nix
+++ b/pkgs/development/python-modules/transformers/default.nix
@@ -2,6 +2,7 @@
 , stdenv
 , fetchFromGitHub
 , boto3
+, cookiecutter
 , filelock
 , regex
 , requests
@@ -9,7 +10,6 @@
 , parameterized
 , protobuf
 , sacremoses
-, sentencepiece
 , timeout-decorator
 , tokenizers
 , tqdm
@@ -18,23 +18,23 @@
 
 buildPythonPackage rec {
   pname = "transformers";
-  version = "3.4.0";
+  version = "4.0.0";
 
   src = fetchFromGitHub {
     owner = "huggingface";
     repo = pname;
     rev = "v${version}";
-    sha256 = "1v09gryxsg57d2cjwagna1535m8mbxlazdbhsww210lxa818m5qj";
+    sha256 = "17djq32pq8d6vqip7i9pda0ldigmzckbbcd278llmpxdriqd4llg";
   };
 
   propagatedBuildInputs = [
+    cookiecutter
     filelock
     numpy
     protobuf
     regex
     requests
     sacremoses
-    sentencepiece
     tokenizers
     tqdm
   ];
@@ -45,6 +45,11 @@ buildPythonPackage rec {
     timeout-decorator
   ];
 
+  postPatch = ''
+    substituteInPlace setup.py \
+      --replace "tokenizers == 0.9.4" "tokenizers"
+  '';
+
   preCheck = ''
     export HOME="$TMPDIR"
 
@@ -65,11 +70,13 @@ buildPythonPackage rec {
   disabledTests = [
     "BlenderbotSmallTokenizerTest"
     "Blenderbot3BTokenizerTests"
+    "GetFromCacheTests"
     "TokenizationTest"
     "TestTokenizationBart"
     "test_all_tokenizers"
     "test_batch_encoding_is_fast"
     "test_batch_encoding_pickle"
+    "test_batch_encoding_word_to_tokens"
     "test_config_from_model_shortcut"
     "test_config_model_type_from_model_identifier"
     "test_from_pretrained_use_fast_toggle"
@@ -84,6 +91,7 @@ buildPythonPackage rec {
     "test_tokenizer_from_pretrained"
     "test_tokenizer_from_tokenizer_class"
     "test_tokenizer_identifier_with_correct_config"
+    "test_tokenizer_identifier_non_existent"
   ];
 
   meta = with stdenv.lib; {