


'''simple docstring'''







# Copyright 2022 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from typing import List







def _UpperCamelCase	(					__A				,      __A   )							->		Optional[int]:


      '''simple docstring'''



      if name != "training_function" and name != "main":
            raise ValueError(F'''Incorrect function name passed: {name}, choose either \'main\' or \'training_function\''''   )
      UpperCamelCase__ ,     UpperCamelCase__  =    [], False
      for line in lines:
            if not found_start and F'''def {name}''' in line:
                  UpperCamelCase__  =    True
                  good_lines.append(__A   )
                  continue
            if found_start:
                  if name == "training_function" and "def main" in line:
                        return good_lines
                  if name == "main" and "if __name__" in line:
                        return good_lines
                  good_lines.append(__A   )







def _UpperCamelCase	(					__A   )							->		Optional[Any]:


      '''simple docstring'''



      return [line for line in lines if not line.lstrip().startswith("#"   ) and line != "\n"]







def _UpperCamelCase	(					__A				,      __A				,      __A				,      __A = None   )							->		str:


      '''simple docstring'''



      with open(__A				,      "r"   ) as f:
            UpperCamelCase__  =    f.readlines()
      with open(os.path.abspath(os.path.join("examples"				,      "nlp_example.py"   )   )				,      "r"   ) as f:
            UpperCamelCase__  =    f.readlines()
      with open(__A				,      "r"   ) as f:
            UpperCamelCase__  =    f.readlines()
      if secondary_filename is not None:
            with open(__A				,      "r"   ) as f:
                  UpperCamelCase__  =    f.readlines()

    # This is our base, we remove all the code from here in our `full_filename` and `feature_filename` to find the new content
      if parser_only:
            UpperCamelCase__  =    clean_lines(get_function_contents_by_name(__A				,      "main"   )   )
            UpperCamelCase__  =    clean_lines(get_function_contents_by_name(__A				,      "main"   )   )
            UpperCamelCase__  =    clean_lines(get_function_contents_by_name(__A				,      "main"   )   )
            if secondary_filename is not None:
                  UpperCamelCase__  =    clean_lines(get_function_contents_by_name(__A				,      "main"   )   )
      else:
            UpperCamelCase__  =    clean_lines(get_function_contents_by_name(__A				,      "training_function"   )   )
            UpperCamelCase__  =    clean_lines(get_function_contents_by_name(__A				,      "training_function"   )   )
            UpperCamelCase__  =    clean_lines(get_function_contents_by_name(__A				,      "training_function"   )   )
            if secondary_filename is not None:
                  UpperCamelCase__  =    clean_lines(
                      get_function_contents_by_name(__A				,      "training_function"   )   )

      UpperCamelCase__  =    "train_dataloader, eval_dataloader = get_dataloaders(accelerator, batch_size)\n"

      # Specific code in our script that differs from the full version, aka what is new
      UpperCamelCase__  =    []
      UpperCamelCase__  =    []  # We keep track of the idxs just in case it's a repeated statement
      UpperCamelCase__  =    iter(__A   )
      for i in range(len(__A   ) - 1   ):
            if i not in passed_idxs:
                  UpperCamelCase__  =    next(__A   )
                  if (line not in full_file_func) and (line.lstrip() != _dl_line):
                        if "TESTING_MOCKED_DATALOADERS" not in line:
                              new_feature_code.append(__A   )
                              passed_idxs.append(__A   )
                        else:
                              # Skip over the `config['num_epochs'] = 2` statement
                              UpperCamelCase__  =    next(__A   )

    # Extract out just the new parts from the full_file_training_func
      UpperCamelCase__  =    []
      UpperCamelCase__  =    []  # We keep track of the idxs just in case it's a repeated statement
      for i, line in enumerate(__A   ):
            if i not in passed_idxs:
                  if (line not in full_file_func) and (line.lstrip() != _dl_line):
                        if "TESTING_MOCKED_DATALOADERS" not in line:
                              new_full_example_parts.append(__A   )
                              passed_idxs.append(__A   )

    # Finally, get the overall diff
      UpperCamelCase__  =    [line for line in new_feature_code if line not in new_full_example_parts]
      if secondary_filename is not None:
            UpperCamelCase__  =    [line for line in full_file_contents if line not in secondary_file_func]
            UpperCamelCase__  =    [line for line in diff_from_example if line not in diff_from_two]

      return diff_from_example









'''simple docstring'''







import json
import os
import shutil
import tempfile
import unittest

from transformers import is_speech_available, is_torch_available
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_torch
from transformers.utils import FEATURE_EXTRACTOR_NAME


if is_speech_available() and is_torch_available():
  from transformers import SpeechTaFeatureExtractor, SpeechTaProcessor

  from .test_feature_extraction_speechta import floats_list


a__  : int				=	get_tests_dir('fixtures/test_sentencepiece_bpe_char.model')
@require_torch
class  lowercase_					(   unittest.TestCase       ):


    def        __a							(					self   ):
          UpperCamelCase__  =    tempfile.mkdtemp()

          UpperCamelCase__  =    SpeechTaTokenizer(a   )
          tokenizer.save_pretrained(self.tmpdirname   )

          UpperCamelCase__  =    {
              "feature_size": 1,
              "padding_value": 0.0,
              "sampling_rate": 1_60_00,
              "do_normalize": False,
              "num_mel_bins": 80,
              "hop_length": 16,
              "win_length": 64,
              "win_function": "hann_window",
              "fmin": 80,
              "fmax": 76_00,
              "mel_floor": 1e-10,
              "reduction_factor": 2,
              "return_attention_mask": True,
          }

          UpperCamelCase__  =    os.path.join(self.tmpdirname  , a   )
          with open(self.feature_extraction_file  , "w"  , encoding="utf-8"   ) as fp:
                fp.write(json.dumps(a   ) + "\n"   )


    def        __a							(					self  , **a   ):
          return SpeechTaTokenizer.from_pretrained(self.tmpdirname  , **a   )


    def        __a							(					self  , **a   ):
          return SpeechTaFeatureExtractor.from_pretrained(self.tmpdirname  , **a   )


    def        __a							(					self   ):
          shutil.rmtree(self.tmpdirname   )


    def        __a							(					self   ):
          UpperCamelCase__  =    self.get_tokenizer()
          UpperCamelCase__  =    self.get_feature_extractor()

          UpperCamelCase__  =    SpeechTaProcessor(tokenizer=a  , feature_extractor=a   )

          processor.save_pretrained(self.tmpdirname   )
          UpperCamelCase__  =    SpeechTaProcessor.from_pretrained(self.tmpdirname   )

          self.assertEqual(processor.tokenizer.get_vocab()  , tokenizer.get_vocab()   )
          self.assertIsInstance(processor.tokenizer  , a   )

          self.assertEqual(processor.feature_extractor.to_json_string()  , feature_extractor.to_json_string()   )
          self.assertIsInstance(processor.feature_extractor  , a   )


    def        __a							(					self   ):
          UpperCamelCase__  =    SpeechTaProcessor(tokenizer=self.get_tokenizer()  , feature_extractor=self.get_feature_extractor()   )
          processor.save_pretrained(self.tmpdirname   )

          UpperCamelCase__  =    self.get_tokenizer(bos_token="(BOS)"  , eos_token="(EOS)"   )
          UpperCamelCase__  =    self.get_feature_extractor(do_normalize=a  , padding_value=1.0   )

          UpperCamelCase__  =    SpeechTaProcessor.from_pretrained(
              self.tmpdirname  , bos_token="(BOS)"  , eos_token="(EOS)"  , do_normalize=a  , padding_value=1.0   )

          self.assertEqual(processor.tokenizer.get_vocab()  , tokenizer_add_kwargs.get_vocab()   )
          self.assertIsInstance(processor.tokenizer  , a   )

          self.assertEqual(processor.feature_extractor.to_json_string()  , feature_extractor_add_kwargs.to_json_string()   )
          self.assertIsInstance(processor.feature_extractor  , a   )


    def        __a							(					self   ):
          UpperCamelCase__  =    self.get_feature_extractor()
          UpperCamelCase__  =    self.get_tokenizer()

          UpperCamelCase__  =    SpeechTaProcessor(tokenizer=a  , feature_extractor=a   )

          UpperCamelCase__  =    floats_list((3, 10_00)   )

          UpperCamelCase__  =    feature_extractor(audio=a  , return_tensors="np"   )
          UpperCamelCase__  =    processor(audio=a  , return_tensors="np"   )

          for key in input_feat_extract.keys():
                self.assertAlmostEqual(input_feat_extract[key].sum()  , input_processor[key].sum()  , delta=1e-2   )


    def        __a							(					self   ):
          UpperCamelCase__  =    self.get_feature_extractor()
          UpperCamelCase__  =    self.get_tokenizer()

          UpperCamelCase__  =    SpeechTaProcessor(tokenizer=a  , feature_extractor=a   )

          UpperCamelCase__  =    floats_list((3, 10_00)   )

          UpperCamelCase__  =    feature_extractor(audio_target=a  , return_tensors="np"   )
          UpperCamelCase__  =    processor(audio_target=a  , return_tensors="np"   )

          for key in input_feat_extract.keys():
                self.assertAlmostEqual(input_feat_extract[key].sum()  , input_processor[key].sum()  , delta=1e-2   )


    def        __a							(					self   ):
          UpperCamelCase__  =    self.get_feature_extractor()
          UpperCamelCase__  =    self.get_tokenizer()

          UpperCamelCase__  =    SpeechTaProcessor(tokenizer=a  , feature_extractor=a   )

          UpperCamelCase__  =    "This is a test string"

          UpperCamelCase__  =    processor(text=a   )
          UpperCamelCase__  =    tokenizer(a   )

          for key in encoded_tok.keys():
                self.assertListEqual(encoded_tok[key]  , encoded_processor[key]   )


    def        __a							(					self   ):
          UpperCamelCase__  =    self.get_feature_extractor()
          UpperCamelCase__  =    self.get_tokenizer()

          UpperCamelCase__  =    SpeechTaProcessor(tokenizer=a  , feature_extractor=a   )

          UpperCamelCase__  =    "This is a test string"

          UpperCamelCase__  =    processor(text_target=a   )
          UpperCamelCase__  =    tokenizer(a   )

          for key in encoded_tok.keys():
                self.assertListEqual(encoded_tok[key]  , encoded_processor[key]   )


    def        __a							(					self   ):
          UpperCamelCase__  =    self.get_feature_extractor()
          UpperCamelCase__  =    self.get_tokenizer()

          UpperCamelCase__  =    SpeechTaProcessor(tokenizer=a  , feature_extractor=a   )

          UpperCamelCase__  =    [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

          UpperCamelCase__  =    processor.batch_decode(a   )
          UpperCamelCase__  =    tokenizer.batch_decode(a   )

          self.assertListEqual(a  , a   )


    def        __a							(					self   ):
          UpperCamelCase__  =    self.get_feature_extractor()
          UpperCamelCase__  =    self.get_tokenizer()

          UpperCamelCase__  =    SpeechTaProcessor(tokenizer=a  , feature_extractor=a   )

          self.assertListEqual(
              processor.model_input_names  , feature_extractor.model_input_names  , msg="`processor` and `feature_extractor` model input names do not match"  , )





