
import os
from typing import Optional, Union

import numpy as np
from huggingface_hub import hf_hub_download

from ... import AutoTokenizer
from ...utils import logging


__a        = 'block_records.npy'


__a        = logging.get_logger(__name__)




def     a						(							snake_case__: str						,       snake_case__: int ):


					'''simple docstring'''
					import tensorflow.compat.va as tf

					lowercase_					  = tf.data.TFRecordDataset(snake_case__						,       buffer_size=512 * 1_024 * 1_024 )
					lowercase_					  = blocks_dataset.batch(snake_case__						,       drop_remainder=snake_case__ )
					lowercase_					  = next(blocks_dataset.take(1 ).as_numpy_iterator() )

					return np_record
class 			lowercase__:



	"""simple docstring"""




	def __init__(	self  :      List[str]	,							SCREAMING_SNAKE_CASE_  :      int	,							SCREAMING_SNAKE_CASE_  :      Tuple	,							SCREAMING_SNAKE_CASE_  :      Dict=2	,							SCREAMING_SNAKE_CASE_  :      Dict=1_0_0_0	,							SCREAMING_SNAKE_CASE_  :      Optional[int]=1_0_0	,							SCREAMING_SNAKE_CASE_  :      List[Any]=1_0_0_0_0_0	,							)	->					Any:
						from scann.scann_ops.py.scann_ops_pybind import builder as Builder

						lowercase_					  = Builder(db=SCREAMING_SNAKE_CASE_	,							num_neighbors=SCREAMING_SNAKE_CASE_	,							distance_measure='''dot_product'''	)
						lowercase_					  = builder.tree(
						    num_leaves=SCREAMING_SNAKE_CASE_	,							num_leaves_to_search=SCREAMING_SNAKE_CASE_	,							training_sample_size=SCREAMING_SNAKE_CASE_	)
						lowercase_					  = builder.score_ah(dimensions_per_block=SCREAMING_SNAKE_CASE_	)

						lowercase_					  = builder.build()




	def 						_lowercase  (	self  :      Optional[Any]	,							SCREAMING_SNAKE_CASE_  :      Dict	)	->					List[str]:
						lowercase_  ,     lowercase_					  = self.searcher.search_batched(question_projection.detach().cpu()	)
						return retrieved_block_ids.astype('''int64'''	)

class 			lowercase__:



	"""simple docstring"""




	def __init__(	self  :      str	,							SCREAMING_SNAKE_CASE_  :      List[str]	,							SCREAMING_SNAKE_CASE_  :      Any	)	->					Optional[Any]:
						super().__init__()
						lowercase_					  = block_records
						lowercase_					  = tokenizer
	def __call__(	self  :      Any	,							SCREAMING_SNAKE_CASE_  :      str	,							SCREAMING_SNAKE_CASE_  :      Union[str, Any]	,							SCREAMING_SNAKE_CASE_  :      Optional[int]	,							SCREAMING_SNAKE_CASE_  :      str=None	,							SCREAMING_SNAKE_CASE_  :      List[Any]="pt"	)	->					Tuple:
						lowercase_					  = np.take(self.block_records	,							indices=SCREAMING_SNAKE_CASE_	,							axis=0	)

						lowercase_					  = self.tokenizer.decode(question_input_ids[0]	,							skip_special_tokens=SCREAMING_SNAKE_CASE_	)

						lowercase_					  = []
						lowercase_					  = []
						for retrieved_block in retrieved_blocks:
											text.append(SCREAMING_SNAKE_CASE_	)
											text_pair.append(retrieved_block.decode()	)

						lowercase_					  = self.tokenizer(
						    SCREAMING_SNAKE_CASE_	,							SCREAMING_SNAKE_CASE_	,							padding=SCREAMING_SNAKE_CASE_	,							truncation=SCREAMING_SNAKE_CASE_	,							return_special_tokens_mask=SCREAMING_SNAKE_CASE_	,							max_length=SCREAMING_SNAKE_CASE_	)
						lowercase_					  = concat_inputs.convert_to_tensors(SCREAMING_SNAKE_CASE_	)

						if answer_ids is not None:
											return self.block_has_answer(SCREAMING_SNAKE_CASE_	,							SCREAMING_SNAKE_CASE_	) + (concat_inputs_tensors,)
						else:
											return (None, None, None, concat_inputs_tensors)
	@classmethod
	def 						_lowercase  (	cls  :      Tuple	,							SCREAMING_SNAKE_CASE_  :      Optional[Union[str, os.PathLike]]	,							*SCREAMING_SNAKE_CASE_  :      List[Any]	,							**SCREAMING_SNAKE_CASE_  :      Dict	)	->					List[str]:
						if os.path.isdir(SCREAMING_SNAKE_CASE_	):
											lowercase_					  = os.path.join(SCREAMING_SNAKE_CASE_	,							_REALM_BLOCK_RECORDS_FILENAME	)
						else:
											lowercase_					  = hf_hub_download(
											    repo_id=SCREAMING_SNAKE_CASE_	,							filename=_REALM_BLOCK_RECORDS_FILENAME	,							**SCREAMING_SNAKE_CASE_	)
						lowercase_					  = np.load(SCREAMING_SNAKE_CASE_	,							allow_pickle=SCREAMING_SNAKE_CASE_	)

						lowercase_					  = AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_	,							*SCREAMING_SNAKE_CASE_	,							**SCREAMING_SNAKE_CASE_	)

						return cls(SCREAMING_SNAKE_CASE_	,							SCREAMING_SNAKE_CASE_	)
	def 						_lowercase  (	self  :      int	,							SCREAMING_SNAKE_CASE_  :      Any	)	->					Optional[Any]:
						# save block records
						np.save(os.path.join(SCREAMING_SNAKE_CASE_	,							_REALM_BLOCK_RECORDS_FILENAME	)	,							self.block_records	)
						# save tokenizer
						self.tokenizer.save_pretrained(SCREAMING_SNAKE_CASE_	)




	def 						_lowercase  (	self  :      Dict	,							SCREAMING_SNAKE_CASE_  :      Optional[Any]	,							SCREAMING_SNAKE_CASE_  :      List[Any]	)	->					Optional[Any]:
						lowercase_					  = []
						lowercase_					  = []
						lowercase_					  = []
						lowercase_					  = 0

						for input_id in concat_inputs.input_ids:
											lowercase_					  = input_id.tolist()
											# Check answers between two [SEP] tokens
											lowercase_					  = input_id_list.index(self.tokenizer.sep_token_id	)
											lowercase_					  = first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id	)

											start_pos.append([]	)
											end_pos.append([]	)
											for answer in answer_ids:
																for idx in range(first_sep_idx + 1	,							SCREAMING_SNAKE_CASE_	):
																					if answer[0] == input_id_list[idx]:
																										if input_id_list[idx : idx + len(SCREAMING_SNAKE_CASE_	)] == answer:
																															start_pos[-1].append(SCREAMING_SNAKE_CASE_	)
																															end_pos[-1].append(idx + len(SCREAMING_SNAKE_CASE_	) - 1	)

											if len(start_pos[-1]	) == 0:
																has_answers.append(SCREAMING_SNAKE_CASE_	)
											else:
																has_answers.append(SCREAMING_SNAKE_CASE_	)
																if len(start_pos[-1]	) > max_answers:
																					lowercase_					  = len(start_pos[-1]	)

        # Pad -1 to max_answers
						for start_pos_, end_pos_ in zip(SCREAMING_SNAKE_CASE_	,							SCREAMING_SNAKE_CASE_	):
											if len(start_pos_	) < max_answers:
																lowercase_					  = [-1] * (max_answers - len(start_pos_	))
																start_pos_ += padded
																end_pos_ += padded
						return has_answers, start_pos, end_pos






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


__a        = {
    'configuration_donut_swin': ['DONUT_SWIN_PRETRAINED_CONFIG_ARCHIVE_MAP', 'DonutSwinConfig'],
    'processing_donut': ['DonutProcessor'],
}

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						__a        = [
						    'DONUT_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST',
						    'DonutSwinModel',
						    'DonutSwinPreTrainedModel',
						]

try:
						if not is_vision_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						__a        = ['DonutFeatureExtractor']
						__a        = ['DonutImageProcessor']


if TYPE_CHECKING:
						from .configuration_donut_swin import DONUT_SWIN_PRETRAINED_CONFIG_ARCHIVE_MAP, DonutSwinConfig
						from .processing_donut import DonutProcessor

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_donut_swin import (
												    DONUT_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST,
												    DonutSwinModel,
												    DonutSwinPreTrainedModel,
												)

						try:
												if not is_vision_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .feature_extraction_donut import DonutFeatureExtractor
												from .image_processing_donut import DonutImageProcessor

else:
						import sys

						__a        = _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)




