
"""simple docstring"""




import importlib
import importlib.metadata
import os
import warnings
from distutils.util import strtobool
from functools import lru_cache

import torch
from packaging import version
from packaging.version import parse

from .environment import parse_flag_from_env
from .versions import compare_versions, is_torch_version


try:
	import torch_xla.core.xla_model as xm  # noqa: F401

	snake_case			       =		True
except ImportError:
	snake_case			       =		False


# Cache this result has it's a C FFI call which can be pretty time-consuming
snake_case			       =		torch.distributed.is_available()

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_      ):
					# Check we're not importing a "pkg_name" directory somewhere but the actual library by trying to grab the version
					SCREAMING_SNAKE_CASE           =     importlib.util.find_spec(SCREAMING_SNAKE_CASE_      ) is not None
					if package_exists:
										try:
															SCREAMING_SNAKE_CASE           =     importlib.metadata.metadata(SCREAMING_SNAKE_CASE_      )
															return True
										except importlib.metadata.PackageNotFoundError:
															return False

def   UpperCamelCase_  (		):
					return _torch_distributed_available

def   UpperCamelCase_  (		):
					try:
										pass
					except ImportError:
										print(
										    'Intel(R) oneCCL Bindings for PyTorch* is required to run DDP on Intel(R) GPUs, but it is not'
										    ' detected. If you see \"ValueError: Invalid backend: \'ccl\'\" error, please install Intel(R) oneCCL'
										    ' Bindings for PyTorch*.'      )
					return (
					    importlib.util.find_spec('torch_ccl'      ) is not None
					    or importlib.util.find_spec('oneccl_bindings_for_pytorch'      ) is not None
					)

def   UpperCamelCase_  (		):
					return importlib.metadata.version('oneccl_bind_pt'      )

def   UpperCamelCase_  (		):
					return _is_package_available('transformer_engine'      )

@lru_cache
def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_=True      ):
					# Due to bugs on the amp series GPUs, we disable torch-xla on them
					if torch.cuda.is_available():
										return False
					if _tpu_available and check_device:
										try:
															# Will raise a RuntimeError if no XLA configuration is found
															SCREAMING_SNAKE_CASE           =     xm.xla_device()
															return True
										except RuntimeError:
															return False
					return _tpu_available

def   UpperCamelCase_  (		):
					return _is_package_available('deepspeed'      )

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_=False      ):
					if is_tpu_available():
										return not ignore_tpu
					if torch.cuda.is_available():
										return torch.cuda.is_bfaa_supported()
					if is_npu_available():
										return False
					return True

def   UpperCamelCase_  (		):
					SCREAMING_SNAKE_CASE           =     _is_package_available('bitsandbytes'      )
					if package_exists:
										SCREAMING_SNAKE_CASE           =     version.parse(importlib.metadata.version('bitsandbytes'      )      )
										return compare_versions(SCREAMING_SNAKE_CASE_,					'>=',					'0.39.0'      )
					return False

def   UpperCamelCase_  (		):
					SCREAMING_SNAKE_CASE           =     _is_package_available('bitsandbytes'      )
					if package_exists:
										SCREAMING_SNAKE_CASE           =     version.parse(importlib.metadata.version('bitsandbytes'      )      )
										return compare_versions(SCREAMING_SNAKE_CASE_,					'>=',					'0.37.2'      )
					return False

def   UpperCamelCase_  (		):
					return _is_package_available('bitsandbytes'      )

def   UpperCamelCase_  (		):
					if strtobool(os.environ.get('ACCELERATE_USE_MEGATRON_LM',					'False'      )      ) == 1:
										SCREAMING_SNAKE_CASE           =     importlib.util.find_spec('megatron'      ) is not None
										if package_exists:
															try:
																				SCREAMING_SNAKE_CASE           =     parse(importlib.metadata.version('megatron-lm'      )      )
																				return compare_versions(SCREAMING_SNAKE_CASE_,					'>=',					'2.2.0'      )
															except Exception as e:
																				warnings.warn(f'''Parse Megatron version failed. Exception:{e}'''      )
																				return False

def   UpperCamelCase_  (		):
					return _is_package_available('safetensors'      )

def   UpperCamelCase_  (		):
					return _is_package_available('transformers'      )

def   UpperCamelCase_  (		):
					return _is_package_available('datasets'      )

def   UpperCamelCase_  (		):
					return _is_package_available('aim'      )

def   UpperCamelCase_  (		):
					return _is_package_available('tensorboard'      ) or _is_package_available('tensorboardX'      )

def   UpperCamelCase_  (		):
					return _is_package_available('wandb'      )

def   UpperCamelCase_  (		):
					return _is_package_available('comet_ml'      )

def   UpperCamelCase_  (		):
					return _is_package_available('boto3'      )

def   UpperCamelCase_  (		):
					if _is_package_available('rich'      ):
										if "ACCELERATE_DISABLE_RICH" in os.environ:
															warnings.warn(
															    '`ACCELERATE_DISABLE_RICH` is deprecated and will be removed in v0.22.0 and deactivated by default. Please use `ACCELERATE_ENABLE_RICH` if you wish to use `rich`.'      )
															return not parse_flag_from_env('ACCELERATE_DISABLE_RICH',					SCREAMING_SNAKE_CASE_      )
										return parse_flag_from_env('ACCELERATE_ENABLE_RICH',					SCREAMING_SNAKE_CASE_      )
					return False

def   UpperCamelCase_  (		):
					return _is_package_available('sagemaker'      )

def   UpperCamelCase_  (		):
					return _is_package_available('tqdm'      )

def   UpperCamelCase_  (		):
					return _is_package_available('mlflow'      )

def   UpperCamelCase_  (		):
					return is_torch_version('>=',					'1.12'      ) and torch.backends.mps.is_available() and torch.backends.mps.is_built()

def   UpperCamelCase_  (		):
					def get_major_and_minor_from_version(SCREAMING_SNAKE_CASE_      ):
										return str(version.parse(SCREAMING_SNAKE_CASE_      ).major      ) + "." + str(version.parse(SCREAMING_SNAKE_CASE_      ).minor      )

					SCREAMING_SNAKE_CASE           =     importlib.metadata.version('torch'      )
					if importlib.util.find_spec('intel_extension_for_pytorch'      ) is None:
										return False
					SCREAMING_SNAKE_CASE           =     'N/A'
					try:
										SCREAMING_SNAKE_CASE           =     importlib.metadata.version('intel_extension_for_pytorch'      )
					except importlib.metadata.PackageNotFoundError:
										return False
					SCREAMING_SNAKE_CASE           =     get_major_and_minor_from_version(_torch_version      )
					SCREAMING_SNAKE_CASE           =     get_major_and_minor_from_version(_ipex_version      )
					if torch_major_and_minor != ipex_major_and_minor:
										warnings.warn(
										    f'''Intel Extension for PyTorch {ipex_major_and_minor} needs to work with PyTorch {ipex_major_and_minor}.*,'''
										    f''' but PyTorch {_torch_version} is found. Please switch to the matching version and run again.'''      )
										return False
					return True

@lru_cache
def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_=False      ):
					if importlib.util.find_spec('torch'      ) is None or importlib.util.find_spec('torch_npu'      ) is None:
										return False

					import torch
					import torch_npu  # noqa: F401

					if check_device:
										try:
															# Will raise a RuntimeError if no NPU is found
															SCREAMING_SNAKE_CASE           =     torch.npu.device_count()
															return torch.npu.is_available()
										except RuntimeError:
															return False
					return hasattr(SCREAMING_SNAKE_CASE_,					'npu'      ) and torch.npu.is_available()



@lru_cache
def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_=False      ):
					if not parse_flag_from_env('ACCELERATE_USE_XPU',					default=SCREAMING_SNAKE_CASE_      ):
										return False
					if is_ipex_available():
										import torch

										if is_torch_version('<=',					'1.12'      ):
															return False
					else:
										return False

					import intel_extension_for_pytorch  # noqa: F401

					if check_device:
										try:
															# Will raise a RuntimeError if no XPU  is found
															SCREAMING_SNAKE_CASE           =     torch.xpu.device_count()
															return torch.xpu.is_available()
										except RuntimeError:
															return False
					return hasattr(SCREAMING_SNAKE_CASE_,					'xpu'      ) and torch.xpu.is_available()


"""simple docstring"""




import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


snake_case			       =		logging.getLogger(__name__)

def   UpperCamelCase_  (		):
					SCREAMING_SNAKE_CASE           =     argparse.ArgumentParser(description='Train a unigram tokenizer on the wikitext dataset.'      )
					parser.add_argument(
					    '--dataset_name',					type=SCREAMING_SNAKE_CASE_,					default='wikitext',					help='Name of the training. Explore datasets at: hf.co/datasets.',					)
					parser.add_argument(
					    '--dataset_config',					type=SCREAMING_SNAKE_CASE_,					default='wikitext-103-raw-v1',					help='Configuration name of the dataset.'      )
					parser.add_argument(
					    '--batch_size',					type=SCREAMING_SNAKE_CASE_,					default=1_0_0_0,					help='Batch size during training.',					)
					parser.add_argument(
					    '--vocab_size',					type=SCREAMING_SNAKE_CASE_,					default=1_0_0_4_8,					help='Size of the desired vocabulary.',					)
					parser.add_argument(
					    '--limit',					default=SCREAMING_SNAKE_CASE_,					type=SCREAMING_SNAKE_CASE_,					help='Limit the number of shards (used for debugging).',					)
					parser.add_argument(
					    '--export_to_hub',					action='store_true',					)

					SCREAMING_SNAKE_CASE           =     parser.parse_args()
					return args

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_      ):
					SCREAMING_SNAKE_CASE           =     datasets.load_dataset(args.dataset_name,					args.dataset_config,					split='train'      )

					if args.limit is not None:
										SCREAMING_SNAKE_CASE           =     min(len(SCREAMING_SNAKE_CASE_      ),					args.limit      )
										SCREAMING_SNAKE_CASE           =     dataset.select(range(SCREAMING_SNAKE_CASE_      )      )
										logger.info(f'''Limiting the dataset to {args.limit} entries.'''      )

					def batch_iterator():
										for i in range(0,					len(SCREAMING_SNAKE_CASE_      ),					args.batch_size      ):
															yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
					SCREAMING_SNAKE_CASE           =     Tokenizer(Unigram()      )
					SCREAMING_SNAKE_CASE           =     normalizers.Sequence([normalizers.Replace('``',					'"'      ), normalizers.Replace('\'\'',					'"'      )]      )
					SCREAMING_SNAKE_CASE           =     pre_tokenizers.Metaspace()

					# Prepare the trainer.
					SCREAMING_SNAKE_CASE           =     UnigramTrainer(
					    unk_token='<unk>',					special_tokens=['[CLS]', '[SEP]', '<unk>', '<pad>', '[MASK]'],					vocab_size=args.vocab_size,					)

					logger.info('Training the tokenizer.'      )
					tokenizer.train_from_iterator(batch_iterator(),					trainer=SCREAMING_SNAKE_CASE_      )
					logger.info('Tokenizer training complete!'      )

					SCREAMING_SNAKE_CASE           =     tokenizer.token_to_id('[CLS]'      )
					SCREAMING_SNAKE_CASE           =     tokenizer.token_to_id('[SEP]'      )
					SCREAMING_SNAKE_CASE           =     processors.TemplateProcessing(
					    single='[CLS]:0 $A:0 [SEP]:0',					pair='[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1',					special_tokens=[
					        ('[CLS]', cls_token_id),
					        ('[SEP]', sep_token_id),
					    ],					)
					SCREAMING_SNAKE_CASE           =     decoders.Metaspace()

					if args.export_to_hub:
										logger.info('Exporting the trained tokenzier to Hub.'      )
										SCREAMING_SNAKE_CASE           =     AlbertTokenizerFast(tokenizer_object=SCREAMING_SNAKE_CASE_      )
										new_tokenizer.push_to_hub('unigram-tokenizer-dataset'      )


if __name__ == "__main__":
	snake_case			       =		parse_args()
	main(args)
