



import subprocess
import sys
import warnings
from argparse import ArgumentParser
from pathlib import Path

from packaging import version

from .. import AutoFeatureExtractor, AutoImageProcessor, AutoProcessor, AutoTokenizer
from ..utils import logging
from ..utils.import_utils import is_optimum_available
from .convert import export, validate_model_outputs
from .features import FeaturesManager
from .utils import get_preprocessor


a_       :int				  =				"1.5.0"

a_       :Optional[int]				  =				["vision-encoder-decoder"]
def 	lowercase_							(A			:			Optional[int]				):
       if is_optimum_available():
              from optimum.version import __version__ as optimum_version

              snake_case__							:     Union[str, Any]            =     version.parse(A				)
              if parsed_optimum_version < version.parse(A				):
                     raise RuntimeError(
                         F'''transformers.onnx requires optimum >= {MIN_OPTIMUM_VERSION} but {optimum_version} is installed. You '''
                         'can upgrade optimum by running: pip install -U optimum[exporters]'				)
       else:
              raise RuntimeError(
                  'transformers.onnx requires optimum to run, you can install the library by running: pip install '
                  'optimum[exporters]'				)
       snake_case__							:     List[Any]            =     [
           sys.executable,
           '''-m''',
           '''optimum.exporters.onnx''',
           F'''--model {args.model}''',
           F'''--task {args.feature}''',
           F'''--framework {args.framework}''' if args.framework is not None else '''''',
           F'''{args.output}''',
       ]
       snake_case__							:     Tuple            =     subprocess.Popen(' '.join(A				)		, stdout=subprocess.PIPE		, shell=A				)
       proc.wait()

       logger.info(
           'The export was done by optimum.exporters.onnx. We recommend using to use this package directly in future, as '
           'transformers.onnx is deprecated, and will be removed in v5. You can find more information here: '
           'https://huggingface.co/docs/optimum/exporters/onnx/usage_guides/export_a_model.'				)
def 	lowercase_							(A			:			Optional[int]				):
       snake_case__							:     Union[str, Any]            =     args.output if args.output.is_file() else args.output.joinpath('model.onnx'				)
       if not args.output.parent.exists():
              args.output.parent.mkdir(parents=A				)

       # Allocate the model
       snake_case__							:     Optional[Any]            =     FeaturesManager.get_model_from_feature(
           args.feature		, args.model		, framework=args.framework		, cache_dir=args.cache_dir				)

       snake_case__							:     Optional[Any]            =     FeaturesManager.check_supported_model_or_raise(A		, feature=args.feature				)
       snake_case__							:     List[str]            =     model_onnx_config(model.config				)

       if model_kind in ENCODER_DECODER_MODELS:
              snake_case__							:     Any            =     model.get_encoder()
              snake_case__							:     Optional[int]            =     model.get_decoder()

              snake_case__							:     str            =     onnx_config.get_encoder_config(encoder_model.config				)
              snake_case__							:     Union[str, Any]            =     onnx_config.get_decoder_config(
                  encoder_model.config		, decoder_model.config		, feature=args.feature				)

              if args.opset is None:
                     snake_case__							:     List[str]            =     max(encoder_onnx_config.default_onnx_opset		, decoder_onnx_config.default_onnx_opset				)

              if args.opset < min(encoder_onnx_config.default_onnx_opset		, decoder_onnx_config.default_onnx_opset				):
                     raise ValueError(
                         F'''Opset {args.opset} is not sufficient to export {model_kind}. At least '''
                         F''' {min(encoder_onnx_config.default_onnx_opset		, decoder_onnx_config.default_onnx_opset				)} is required.'''				)

              snake_case__							:     str            =     AutoFeatureExtractor.from_pretrained(args.model				)

              snake_case__							:     Optional[Any]            =     export(
                  A		, A		, A		, args.opset		, args.output.parent.joinpath('encoder_model.onnx'				)		, )

              validate_model_outputs(
                  A		, A		, A		, args.output.parent.joinpath('encoder_model.onnx'				)		, A		, args.atol if args.atol else encoder_onnx_config.atol_for_validation		, )

              snake_case__							:     str            =     AutoTokenizer.from_pretrained(args.model				)

              snake_case__							:     List[str]            =     export(
                  A		, A		, A		, args.opset		, args.output.parent.joinpath('decoder_model.onnx'				)		, )

              validate_model_outputs(
                  A		, A		, A		, args.output.parent.joinpath('decoder_model.onnx'				)		, A		, args.atol if args.atol else decoder_onnx_config.atol_for_validation		, )
              logger.info(
                  F'''All good, model saved at: {args.output.parent.joinpath("encoder_model.onnx"				).as_posix()},'''
                  F''' {args.output.parent.joinpath("decoder_model.onnx"				).as_posix()}'''				)

       else:
              # Instantiate the appropriate preprocessor
              if args.preprocessor == "auto":
                     snake_case__							:     List[Any]            =     get_preprocessor(args.model				)
              elif args.preprocessor == "tokenizer":
                     snake_case__							:     Any            =     AutoTokenizer.from_pretrained(args.model				)
              elif args.preprocessor == "image_processor":
                     snake_case__							:     Dict            =     AutoImageProcessor.from_pretrained(args.model				)
              elif args.preprocessor == "feature_extractor":
                     snake_case__							:     List[str]            =     AutoFeatureExtractor.from_pretrained(args.model				)
              elif args.preprocessor == "processor":
                     snake_case__							:     Optional[Any]            =     AutoProcessor.from_pretrained(args.model				)
              else:
                     raise ValueError(F'''Unknown preprocessor type \'{args.preprocessor}\''''				)

              # Ensure the requested opset is sufficient
              if args.opset is None:
                     snake_case__							:     List[Any]            =     onnx_config.default_onnx_opset

              if args.opset < onnx_config.default_onnx_opset:
                     raise ValueError(
                         F'''Opset {args.opset} is not sufficient to export {model_kind}. '''
                         F'''At least  {onnx_config.default_onnx_opset} is required.'''				)

              snake_case__							:     Optional[Any]            =     export(
                  A		, A		, A		, args.opset		, args.output		, )

              if args.atol is None:
                     snake_case__							:     Optional[int]            =     onnx_config.atol_for_validation

              validate_model_outputs(A		, A		, A		, args.output		, A		, args.atol				)
              logger.info(F'''All good, model saved at: {args.output.as_posix()}'''				)
              warnings.warn(
                  'The export was done by transformers.onnx which is deprecated and will be removed in v5. We recommend'
                  ' using optimum.exporters.onnx in future. You can find more information here:'
                  ' https://huggingface.co/docs/optimum/exporters/onnx/usage_guides/export_a_model.'		, A		, )
def 	lowercase_							():
       snake_case__							:     str            =     ArgumentParser('Hugging Face Transformers ONNX exporter'				)
       parser.add_argument(
           '-m'		, '--model'		, type=A		, required=A		, help='Model ID on huggingface.co or path on disk to load model from.'				)
       parser.add_argument(
           '--feature'		, default='default'		, help='The type of features to export the model with.'		, )
       parser.add_argument('--opset'		, type=A		, default=A		, help='ONNX opset version to export the model with.'				)
       parser.add_argument(
           '--atol'		, type=A		, default=A		, help='Absolute difference tolerance when validating the model.'				)
       parser.add_argument(
           '--framework'		, type=A		, choices=['pt', 'tf']		, default=A		, help=(
               'The framework to use for the ONNX export.'
               ' If not provided, will attempt to use the local checkpoint\'s original framework'
               ' or what is available in the environment.'
           )		, )
       parser.add_argument('output'		, type=A		, help='Path indicating where to store generated ONNX model.'				)
       parser.add_argument('--cache_dir'		, type=A		, default=A		, help='Path indicating where to store cache.'				)
       parser.add_argument(
           '--preprocessor'		, type=A		, choices=['auto', 'tokenizer', 'feature_extractor', 'image_processor', 'processor']		, default='auto'		, help='Which type of preprocessor to use. \'auto\' tries to automatically detect it.'		, )
       parser.add_argument(
           '--export_with_transformers'		, action='store_true'		, help=(
               'Whether to use transformers.onnx instead of optimum.exporters.onnx to perform the ONNX export. It can be '
               'useful when exporting a model supported in transformers but not in optimum, otherwise it is not '
               'recommended.'
           )		, )

       snake_case__							:     str            =     parser.parse_args()
       if args.export_with_transformers or not is_optimum_available():
              export_with_transformers(A				)
       else:
              export_with_optimum(A				)


if __name__ == "__main__":
 a_       :Optional[int]				  =				logging.get_logger("transformers.onnx")  # pylint: disable=invalid-name
 logger.setLevel(logging.INFO)
 main()











import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


a_       :Tuple				  =				logging.get_logger(__name__)

a_       :Dict				  =				{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

a_       :List[str]				  =				{
    "vocab_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/vocab.json"},
    "merges_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/merges.txt"},
}

a_       :Union[str, Any]				  =				{
    "openai-gpt": 512,
}
def 	lowercase_							(A			:			Dict				):
       snake_case__							:     Optional[int]            =     text.strip()
       if not text:
              return []
       snake_case__							:     List[str]            =     text.split()
       return tokens




class  snake_case__						(	lowerCAmelCase_					):






     """simple docstring"""


     def __init__(	self  :  List[Any],			_snake_case  :  Any=True,			_snake_case  :  Optional[Any]=None,			_snake_case  :  Dict=True,			_snake_case  :  List[str]=None,			_snake_case  :  List[Any]=True,			)       ->Any:
            if never_split is None:
                   snake_case__							:     int            =     []
            snake_case__							:     Any            =     do_lower_case
            snake_case__							:     Union[str, Any]            =     set(_snake_case )
            snake_case__							:     Union[str, Any]            =     tokenize_chinese_chars
            snake_case__							:     str            =     strip_accents
            snake_case__							:     Optional[Any]            =     do_split_on_punc






     def        lowercase_       (	self  :  Dict,			_snake_case  :  Optional[int],			_snake_case  :  List[Any]=None )       ->List[str]:
            snake_case__							:     Tuple            =     self.never_split.union(set(_snake_case ) ) if never_split else self.never_split
            snake_case__							:     Optional[int]            =     self._clean_text(_snake_case )

            # This was added on November 1st, 2018 for the multilingual and Chinese
            # models. This is also applied to the English models now, but it doesn't
            # matter since the English models were not trained on any Chinese data
            # and generally don't have any Chinese data in them (there are Chinese
            # characters in the vocabulary because Wikipedia does have some Chinese
            # words in the English Wikipedia.).
            if self.tokenize_chinese_chars:
                   snake_case__							:     List[str]            =     self._tokenize_chinese_chars(_snake_case )
            # prevents treating the same character with different unicode codepoints as different characters
            snake_case__							:     Optional[int]            =     unicodedata.normalize('NFC',			_snake_case )
            snake_case__							:     str            =     whitespace_tokenize(_snake_case )
            snake_case__							:     Optional[Any]            =     []
            for token in orig_tokens:
                   if token not in never_split:
                          if self.do_lower_case:
                                 snake_case__							:     int            =     token.lower()
                                 if self.strip_accents is not False:
                                        snake_case__							:     Any            =     self._run_strip_accents(_snake_case )
                          elif self.strip_accents:
                                 snake_case__							:     Optional[Any]            =     self._run_strip_accents(_snake_case )
                   split_tokens.extend(self._run_split_on_punc(_snake_case,			_snake_case ) )

            snake_case__							:     Dict            =     whitespace_tokenize(' '.join(_snake_case ) )
            return output_tokens






     def        lowercase_       (	self  :  int,			_snake_case  :  List[str] )       ->int:
            snake_case__							:     Dict            =     unicodedata.normalize('NFD',			_snake_case )
            snake_case__							:     Dict            =     []
            for char in text:
                   snake_case__							:     List[Any]            =     unicodedata.category(_snake_case )
                   if cat == "Mn":
                          continue
                   output.append(_snake_case )
            return "".join(_snake_case )






     def        lowercase_       (	self  :  str,			_snake_case  :  Dict,			_snake_case  :  Optional[int]=None )       ->Dict:
            if not self.do_split_on_punc or (never_split is not None and text in never_split):
                   return [text]
            snake_case__							:     Dict            =     list(_snake_case )
            snake_case__							:     List[str]            =     0
            snake_case__							:     Union[str, Any]            =     True
            snake_case__							:     Tuple            =     []
            while i < len(_snake_case ):
                   snake_case__							:     Any            =     chars[i]
                   if _is_punctuation(_snake_case ):
                          output.append([char] )
                          snake_case__							:     int            =     True
                   else:
                          if start_new_word:
                                 output.append([] )
                          snake_case__							:     Union[str, Any]            =     False
                          output[-1].append(_snake_case )
                   i += 1

            return ["".join(_snake_case ) for x in output]






     def        lowercase_       (	self  :  Any,			_snake_case  :  int )       ->Any:
            snake_case__							:     Dict            =     []
            for char in text:
                   snake_case__							:     Union[str, Any]            =     ord(_snake_case )
                   if self._is_chinese_char(_snake_case ):
                          output.append(' ' )
                          output.append(_snake_case )
                          output.append(' ' )
                   else:
                          output.append(_snake_case )
            return "".join(_snake_case )






     def        lowercase_       (	self  :  Any,			_snake_case  :  List[str] )       ->Dict:
            # This defines a "chinese character" as anything in the CJK Unicode block:
            #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
            #
            # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
            # despite its name. The modern Korean Hangul alphabet is a different block,
            # as is Japanese Hiragana and Katakana. Those alphabets are used to write
            # space-separated words, so they are not treated specially and handled
            # like the all of the other languages.
            if (
                (cp >= 0X4_E_0_0 and cp <= 0X9_F_F_F)
                or (cp >= 0X3_4_0_0 and cp <= 0X4_D_B_F)  #
                or (cp >= 0X2_0_0_0_0 and cp <= 0X2_A_6_D_F)  #
                or (cp >= 0X2_A_7_0_0 and cp <= 0X2_B_7_3_F)  #
                or (cp >= 0X2_B_7_4_0 and cp <= 0X2_B_8_1_F)  #
                or (cp >= 0X2_B_8_2_0 and cp <= 0X2_C_E_A_F)  #
                or (cp >= 0XF_9_0_0 and cp <= 0XF_A_F_F)
                or (cp >= 0X2_F_8_0_0 and cp <= 0X2_F_A_1_F)  #
            ):  #
                   return True

            return False






     def        lowercase_       (	self  :  List[str],			_snake_case  :  str )       ->Any:
            snake_case__							:     Optional[int]            =     []
            for char in text:
                   snake_case__							:     Any            =     ord(_snake_case )
                   if cp == 0 or cp == 0XF_F_F_D or _is_control(_snake_case ):
                          continue
                   if _is_whitespace(_snake_case ):
                          output.append(' ' )
                   else:
                          output.append(_snake_case )
            return "".join(_snake_case )


def 	lowercase_							(A			:			Dict				):
       snake_case__							:     Dict            =     set()
       snake_case__							:     List[str]            =     word[0]
       for char in word[1:]:
              pairs.add((prev_char, char)				)
              snake_case__							:     str            =     char
       return pairs
def 	lowercase_							(A			:			int				):
       snake_case__							:     Union[str, Any]            =     text.replace('—'		, '-'				)
       snake_case__							:     List[str]            =     text.replace('–'		, '-'				)
       snake_case__							:     List[str]            =     text.replace('―'		, '-'				)
       snake_case__							:     Any            =     text.replace('…'		, '...'				)
       snake_case__							:     Union[str, Any]            =     text.replace('´'		, '\''				)
       snake_case__							:     str            =     re.sub(r'(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'		, r' \1 '		, A				)
       snake_case__							:     int            =     re.sub(r'\s*\n\s*'		, ' \n '		, A				)
       snake_case__							:     Dict            =     re.sub(r'[^\S\n]+'		, ' '		, A				)
       return text.strip()





class  snake_case__						(	lowerCAmelCase_					):






     """simple docstring"""


     _SCREAMING_SNAKE_CASE            =     VOCAB_FILES_NAMES
     _SCREAMING_SNAKE_CASE            =     PRETRAINED_VOCAB_FILES_MAP
     _SCREAMING_SNAKE_CASE            =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _SCREAMING_SNAKE_CASE            =     ["""input_ids""", """attention_mask"""]






     def __init__(	self  :  Optional[Any],			_snake_case  :  Union[str, Any],			_snake_case  :  Tuple,			_snake_case  :  Optional[int]="<unk>",			**_snake_case  :  int )       ->Optional[Any]:
            super().__init__(unk_token=_snake_case,			**_snake_case )

            try:
                   import ftfy
                   from spacy.lang.en import English

                   snake_case__							:     List[Any]            =     English()
                   snake_case__							:     List[Any]            =     _nlp.tokenizer
                   snake_case__							:     List[Any]            =     ftfy.fix_text
            except ImportError:
                   logger.warning('ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.' )
                   snake_case__							:     Union[str, Any]            =     BasicTokenizer(do_lower_case=_snake_case )
                   snake_case__							:     Optional[int]            =     None

            with open(_snake_case,			encoding='utf-8' ) as vocab_handle:
                   snake_case__							:     Union[str, Any]            =     json.load(_snake_case )
            snake_case__							:     List[str]            =     {v: k for k, v in self.encoder.items()}
            with open(_snake_case,			encoding='utf-8' ) as merges_handle:
                   snake_case__							:     int            =     merges_handle.read().split('\n' )[1:-1]
            snake_case__							:     Dict            =     [tuple(merge.split() ) for merge in merges]
            snake_case__							:     Optional[Any]            =     dict(zip(_snake_case,			range(len(_snake_case ) ) ) )
            snake_case__							:     int            =     {}






     @property
     def        lowercase_       (	self  :  Any )       ->str:
            return True






     @property
     def        lowercase_       (	self  :  Dict )       ->str:
            return len(self.encoder )






     def        lowercase_       (	self  :  Dict )       ->Optional[int]:
            return dict(self.encoder,			**self.added_tokens_encoder )






     def        lowercase_       (	self  :  List[Any],			_snake_case  :  Union[str, Any] )       ->Any:
            snake_case__							:     Tuple            =     tuple(token[:-1] ) + (token[-1] + '</w>',)
            if token in self.cache:
                   return self.cache[token]
            snake_case__							:     List[Any]            =     get_pairs(_snake_case )

            if not pairs:
                   return token + "</w>"

            while True:
                   snake_case__							:     Optional[int]            =     min(_snake_case,			key=lambda _snake_case : self.bpe_ranks.get(_snake_case,			float('inf' ) ) )
                   if bigram not in self.bpe_ranks:
                          break
                   snake_case__     ,    snake_case__							:     Tuple            =     bigram
                   snake_case__							:     str            =     []
                   snake_case__							:     int            =     0
                   while i < len(_snake_case ):
                          try:
                                 snake_case__							:     List[str]            =     word.index(_snake_case,			_snake_case )
                          except ValueError:
                                 new_word.extend(word[i:] )
                                 break
                          else:
                                 new_word.extend(word[i:j] )
                                 snake_case__							:     Tuple            =     j

                          if word[i] == first and i < len(_snake_case ) - 1 and word[i + 1] == second:
                                 new_word.append(first + second )
                                 i += 2
                          else:
                                 new_word.append(word[i] )
                                 i += 1
                   snake_case__							:     Optional[int]            =     tuple(_snake_case )
                   snake_case__							:     Union[str, Any]            =     new_word
                   if len(_snake_case ) == 1:
                          break
                   else:
                          snake_case__							:     List[Any]            =     get_pairs(_snake_case )
            snake_case__							:     List[Any]            =     ' '.join(_snake_case )
            if word == "\n  </w>":
                   snake_case__							:     Any            =     '\n</w>'
            snake_case__							:     Optional[int]            =     word
            return word






     def        lowercase_       (	self  :  Union[str, Any],			_snake_case  :  Optional[int] )       ->Dict:
            snake_case__							:     Optional[Any]            =     []
            if self.fix_text is None:
                   # Using BERT's BasicTokenizer
                   snake_case__							:     List[str]            =     self.nlp.tokenize(_snake_case )
                   for token in text:
                          split_tokens.extend(list(self.bpe(_snake_case ).split(' ' ) ) )
            else:
                   # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
                   snake_case__							:     Optional[Any]            =     self.nlp(text_standardize(self.fix_text(_snake_case ) ) )
                   for token in text:
                          split_tokens.extend(list(self.bpe(token.text.lower() ).split(' ' ) ) )
            return split_tokens






     def        lowercase_       (	self  :  Tuple,			_snake_case  :  Union[str, Any] )       ->List[Any]:
            return self.encoder.get(_snake_case,			self.encoder.get(self.unk_token ) )






     def        lowercase_       (	self  :  Tuple,			_snake_case  :  Dict )       ->Any:
            return self.decoder.get(_snake_case,			self.unk_token )






     def        lowercase_       (	self  :  str,			_snake_case  :  List[Any] )       ->List[Any]:
            snake_case__							:     Dict            =     ''.join(_snake_case ).replace('</w>',			' ' ).strip()
            return out_string






     def        lowercase_       (	self  :  Dict,			_snake_case  :  str,			_snake_case  :  Optional[str] = None )       ->Tuple[str]:
            if not os.path.isdir(_snake_case ):
                   logger.error(F'''Vocabulary path ({save_directory}) should be a directory''' )
                   return
            snake_case__							:     Union[str, Any]            =     os.path.join(
                _snake_case,			(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'] )
            snake_case__							:     List[Any]            =     os.path.join(
                _snake_case,			(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file'] )

            with open(_snake_case,			'w',			encoding='utf-8' ) as f:
                   f.write(json.dumps(self.encoder,			indent=2,			sort_keys=_snake_case,			ensure_ascii=_snake_case ) + '\n' )

            snake_case__							:     Union[str, Any]            =     0
            with open(_snake_case,			'w',			encoding='utf-8' ) as writer:
                   writer.write('#version: 0.2\n' )
                   for bpe_tokens, token_index in sorted(self.bpe_ranks.items(),			key=lambda _snake_case : kv[1] ):
                          if index != token_index:
                                 logger.warning(
                                     F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                                     ' Please check that the tokenizer is not corrupted!' )
                                 snake_case__							:     Dict            =     token_index
                          writer.write(' '.join(_snake_case ) + '\n' )
                          index += 1

            return vocab_file, merge_file






