ImportError: cannot import name 'PreTrainedTokenizerBase' from 'transformers.tokenization_utils' (unknown location)

#17
by PankajGaik - opened

recreation:
!pip uninstall -q transformers
!pip install -q nltk sacremoses pandas regex mock mosestokenizer
!pip install -q transformers==4.26.0
!python -m nltk.downloader punkt
!pip install -q IndicTransToolkit@git+https://github.com/VarunGumma/IndicTransToolkit.git

import torch
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
from IndicTransToolkit.processor import IndicProcessor

DEVICE = "cuda" if torch.cuda.is_available() else "cpu"

Error:
ImportError Traceback (most recent call last)
/tmp/ipykernel_9188/3720895640.py in <cell line: 0>()
1 import torch
2 from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
----> 3 from IndicTransToolkit.processor import IndicProcessor
4
5 DEVICE = "cuda" if torch.cuda.is_available() else "cpu"

2 frames
/usr/local/lib/python3.13/dist-packages/IndicTransToolkit/collator.py in IndicDataCollator()
7 class IndicDataCollator:
8 from transformers.utils import PaddingStrategy
----> 9 from transformers.tokenization_utils import PreTrainedTokenizerBase
10 from transformers.data.data_collator import pad_without_fast_tokenizer_warning
11

ImportError: cannot import name 'PreTrainedTokenizerBase' from 'transformers.tokenization_utils' (unknown location)

Sign up or log in to comment