Bump transformers. Fix HF token (#1323)

levkk · web-flow · commit dd7c74909bdf · 2024-02-21T14:30:01.000-08:00
diff --git a/pgml-extension/requirements.linux.txt b/pgml-extension/requirements.linux.txt
@@ -110,7 +110,7 @@ torch==2.1.2
 torchaudio==2.1.2
 torchvision==0.16.2
 tqdm==4.66.1
-transformers==4.36.2
+transformers==4.38.0
 transformers-stream-generator==0.0.4
 triton==2.1.0
 typing-inspect==0.9.0
diff --git a/pgml-extension/src/bindings/transformers/transformers.py b/pgml-extension/src/bindings/transformers/transformers.py
@@ -292,7 +292,7 @@ def __init__(self, model_name, **kwargs):
 
             if "token" in kwargs:
                 self.tokenizer = AutoTokenizer.from_pretrained(
-                    model_name, use_auth_token=kwargs["token"]
+                    model_name, token=kwargs["token"]
                 )
             else:
                 self.tokenizer = AutoTokenizer.from_pretrained(model_name)

Original file line number	Diff line number	Diff line change
`@@ -292,7 +292,7 @@ def __init__(self, model_name, **kwargs):`
`292`	`292`
`293`	`293`	`if "token" in kwargs:`
`294`	`294`	`self.tokenizer = AutoTokenizer.from_pretrained(`
`295`		`- model_name, use_auth_token=kwargs["token"]`
	`295`	`+ model_name, token=kwargs["token"]`
`296`	`296`	`)`
`297`	`297`	`else:`
`298`	`298`	`self.tokenizer = AutoTokenizer.from_pretrained(model_name)`