



import itertools
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa

import datasets
from datasets.table import table_cast


UpperCAmelCase				:							Dict								=datasets.utils.logging.get_logger(__name__)


@dataclass
class  _lowercase				(datasets.BuilderConfig ):
				'''simple docstring'''







				lowercase__				  =  None




class  _lowercase				(datasets.ArrowBasedBuilder ):
				'''simple docstring'''







				lowercase__				  =  ArrowConfig

				def  _lowerCamelCase   (	self      ):

									'''simple docstring'''


									return datasets.DatasetInfo(features=self.config.features      )

				def  _lowerCamelCase   (	self       ,	snake_case__      ):

									'''simple docstring'''


									if not self.config.data_files:
														raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""      )
									UpperCamelCase_											=				dl_manager.download_and_extract(self.config.data_files      )
									if isinstance(snake_case__       ,	(str, list, tuple)      ):
														UpperCamelCase_											=				data_files
														if isinstance(snake_case__       ,	snake_case__      ):
																			UpperCamelCase_											=				[files]
														# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
														UpperCamelCase_											=				[dl_manager.iter_files(snake_case__      ) for file in files]
														return [datasets.SplitGenerator(name=datasets.Split.TRAIN       ,	gen_kwargs={"files": files}      )]
									UpperCamelCase_											=				[]
									for split_name, files in data_files.items():
														if isinstance(snake_case__       ,	snake_case__      ):
																			UpperCamelCase_											=				[files]
														# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
														UpperCamelCase_											=				[dl_manager.iter_files(snake_case__      ) for file in files]
														# Infer features is they are stoed in the arrow schema
														if self.info.features is None:
																			for file in itertools.chain.from_iterable(snake_case__      ):
																								with open(snake_case__       ,	"rb"      ) as f:
																													UpperCamelCase_											=				datasets.Features.from_arrow_schema(pa.ipc.open_stream(snake_case__      ).schema      )
																								break
														splits.append(datasets.SplitGenerator(name=snake_case__       ,	gen_kwargs={"files": files}      )      )
									return splits

				def  _lowerCamelCase   (	self       ,	snake_case__      ):

									'''simple docstring'''


									if self.info.features is not None:
														# more expensive cast to support nested features with keys in a different order
														# allows str <-> int/float or str to Audio for example
														UpperCamelCase_											=				table_cast(snake_case__       ,	self.info.features.arrow_schema      )
									return pa_table

				def  _lowerCamelCase   (	self       ,	snake_case__      ):

									'''simple docstring'''


									for file_idx, file in enumerate(itertools.chain.from_iterable(snake_case__      )      ):
														with open(snake_case__       ,	"rb"      ) as f:
																			try:
																								for batch_idx, record_batch in enumerate(pa.ipc.open_stream(snake_case__      )      ):
																													UpperCamelCase_											=				pa.Table.from_batches([record_batch]      )
																													# Uncomment for debugging (will print the Arrow table size and elements)
																													# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																													# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																													yield F"""{file_idx}_{batch_idx}""", self._cast_table(snake_case__      )
																			except ValueError as e:
																								logger.error(F"""Failed to read file '{file}' with error {type(snake_case__      )}: {e}"""      )
																								raise





import json
import os
import shutil
import tempfile
import unittest

from transformers.models.wavaveca import WavaVecaCTCTokenizer, WavaVecaFeatureExtractor, WavaVecaProcessor
from transformers.models.wavaveca.tokenization_wavaveca import VOCAB_FILES_NAMES
from transformers.utils import FEATURE_EXTRACTOR_NAME

from .test_feature_extraction_wavaveca import floats_list


class  _lowercase				(unittest.TestCase ):
				'''simple docstring'''







				def  _lowerCamelCase   (	self      ):

									'''simple docstring'''


									UpperCamelCase_											=				"<pad> <s> </s> <unk> | E T A O N I H S R D L U M W C F G Y P B V K ' X J Q Z".split(" "      )
									UpperCamelCase_											=				dict(zip(snake_case__       ,	range(len(snake_case__      )      )      )      )

									UpperCamelCase_											=				{
									    "pad_token": "<pad>",
									    "unk_token": "<unk>",
									    "bos_token": "<s>",
									    "eos_token": "</s>",
									}
									UpperCamelCase_											=				{
									    "feature_size": 1,
									    "padding_value": 0.0,
									    "sampling_rate": 1_6000,
									    "return_attention_mask": False,
									    "do_normalize": True,
									}

									UpperCamelCase_											=				tempfile.mkdtemp()
									UpperCamelCase_											=				os.path.join(self.tmpdirname       ,	VOCAB_FILES_NAMES["vocab_file"]      )
									UpperCamelCase_											=				os.path.join(self.tmpdirname       ,	snake_case__      )
									with open(self.vocab_file       ,	"w"       ,	encoding="utf-8"      ) as fp:
														fp.write(json.dumps(snake_case__      ) + "\n"      )

									with open(self.feature_extraction_file       ,	"w"       ,	encoding="utf-8"      ) as fp:
														fp.write(json.dumps(snake_case__      ) + "\n"      )

				def  _lowerCamelCase   (	self       ,	**snake_case__      ):

									'''simple docstring'''


									UpperCamelCase_											=				self.add_kwargs_tokens_map.copy()
									kwargs.update(snake_case__      )
									return WavaVecaCTCTokenizer.from_pretrained(self.tmpdirname       ,	**snake_case__      )

				def  _lowerCamelCase   (	self       ,	**snake_case__      ):

									'''simple docstring'''


									return WavaVecaFeatureExtractor.from_pretrained(self.tmpdirname       ,	**snake_case__      )

				def  _lowerCamelCase   (	self      ):

									'''simple docstring'''


									shutil.rmtree(self.tmpdirname      )

				def  _lowerCamelCase   (	self      ):

									'''simple docstring'''


									UpperCamelCase_											=				self.get_tokenizer()
									UpperCamelCase_											=				self.get_feature_extractor()

									UpperCamelCase_											=				WavaVecaProcessor(tokenizer=snake_case__       ,	feature_extractor=snake_case__      )

									processor.save_pretrained(self.tmpdirname      )
									UpperCamelCase_											=				WavaVecaProcessor.from_pretrained(self.tmpdirname      )

									self.assertEqual(processor.tokenizer.get_vocab()       ,	tokenizer.get_vocab()      )
									self.assertIsInstance(processor.tokenizer       ,	snake_case__      )

									self.assertEqual(processor.feature_extractor.to_json_string()       ,	feature_extractor.to_json_string()      )
									self.assertIsInstance(processor.feature_extractor       ,	snake_case__      )

				def  _lowerCamelCase   (	self      ):

									'''simple docstring'''


									UpperCamelCase_											=				WavaVecaProcessor(tokenizer=self.get_tokenizer()       ,	feature_extractor=self.get_feature_extractor()      )
									processor.save_pretrained(self.tmpdirname      )

									UpperCamelCase_											=				self.get_tokenizer(bos_token="(BOS)"       ,	eos_token="(EOS)"      )
									UpperCamelCase_											=				self.get_feature_extractor(do_normalize=snake_case__       ,	padding_value=1.0      )

									UpperCamelCase_											=				WavaVecaProcessor.from_pretrained(
									    self.tmpdirname       ,	bos_token="(BOS)"       ,	eos_token="(EOS)"       ,	do_normalize=snake_case__       ,	padding_value=1.0      )

									self.assertEqual(processor.tokenizer.get_vocab()       ,	tokenizer_add_kwargs.get_vocab()      )
									self.assertIsInstance(processor.tokenizer       ,	snake_case__      )

									self.assertEqual(processor.feature_extractor.to_json_string()       ,	feature_extractor_add_kwargs.to_json_string()      )
									self.assertIsInstance(processor.feature_extractor       ,	snake_case__      )

				def  _lowerCamelCase   (	self      ):

									'''simple docstring'''


									UpperCamelCase_											=				self.get_feature_extractor()
									UpperCamelCase_											=				self.get_tokenizer()

									UpperCamelCase_											=				WavaVecaProcessor(tokenizer=snake_case__       ,	feature_extractor=snake_case__      )

									UpperCamelCase_											=				floats_list((3, 1000)      )

									UpperCamelCase_											=				feature_extractor(snake_case__       ,	return_tensors="np"      )
									UpperCamelCase_											=				processor(snake_case__       ,	return_tensors="np"      )

									for key in input_feat_extract.keys():
														self.assertAlmostEqual(input_feat_extract[key].sum()       ,	input_processor[key].sum()       ,	delta=1e-2      )

				def  _lowerCamelCase   (	self      ):

									'''simple docstring'''


									UpperCamelCase_											=				self.get_feature_extractor()
									UpperCamelCase_											=				self.get_tokenizer()

									UpperCamelCase_											=				WavaVecaProcessor(tokenizer=snake_case__       ,	feature_extractor=snake_case__      )

									UpperCamelCase_											=				"This is a test string"

									UpperCamelCase_											=				processor(text=snake_case__      )

									UpperCamelCase_											=				tokenizer(snake_case__      )

									for key in encoded_tok.keys():
														self.assertListEqual(encoded_tok[key]       ,	encoded_processor[key]      )

				def  _lowerCamelCase   (	self      ):

									'''simple docstring'''


									UpperCamelCase_											=				self.get_feature_extractor()
									UpperCamelCase_											=				self.get_tokenizer()

									UpperCamelCase_											=				WavaVecaProcessor(tokenizer=snake_case__       ,	feature_extractor=snake_case__      )

									UpperCamelCase_											=				[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

									UpperCamelCase_											=				processor.batch_decode(snake_case__      )
									UpperCamelCase_											=				tokenizer.batch_decode(snake_case__      )

									self.assertListEqual(snake_case__       ,	snake_case__      )

				def  _lowerCamelCase   (	self      ):

									'''simple docstring'''


									UpperCamelCase_											=				self.get_feature_extractor()
									UpperCamelCase_											=				self.get_tokenizer()

									UpperCamelCase_											=				WavaVecaProcessor(tokenizer=snake_case__       ,	feature_extractor=snake_case__      )

									self.assertListEqual(
									    processor.model_input_names       ,	feature_extractor.model_input_names       ,	msg="`processor` and `feature_extractor` model input names do not match"       ,	)
