

"""simple docstring"""




import warnings

import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from sklearn.metrics import pairwise_distances

warnings.filterwarnings("ignore")

A_							:     int     =	"K-MEANS-CLUST/ "

def 		A    (       snake_case__						,			snake_case__						,			snake_case__=None      ):




			'''simple docstring'''


			if seed is not None:  # useful for obtaining consistent results
						np.random.seed(snake_case__      )
			SCREAMING_SNAKE_CASE__									= data.shape[0]  # number of data points

			# Pick K indices from range [0, N).
			SCREAMING_SNAKE_CASE__									= np.random.randint(0						,			snake_case__						,			snake_case__      )

			# Keep centroids as dense format, as many entries will be nonzero due to averaging.
			# As long as at least one document in a cluster contains a word,
			# it will carry a nonzero weight in the TF-IDF vector of the centroid.
			SCREAMING_SNAKE_CASE__									= data[rand_indices, :]

			return centroids

def 		A    (       snake_case__						,			snake_case__      ):




			'''simple docstring'''


			return pairwise_distances(snake_case__						,			snake_case__						,			metric="""euclidean"""      )

def 		A    (       snake_case__						,			snake_case__      ):




			'''simple docstring'''


			SCREAMING_SNAKE_CASE__									= centroid_pairwise_dist(snake_case__						,			snake_case__      )

			# Compute cluster assignments for each data point:
			# Fill in the blank (RHS only)
			SCREAMING_SNAKE_CASE__									= np.argmin(snake_case__						,			axis=1      )

			return cluster_assignment

def 		A    (       snake_case__						,			snake_case__						,			snake_case__      ):




			'''simple docstring'''


			SCREAMING_SNAKE_CASE__									= []
			for i in range(snake_case__      ):
						# Select all data points that belong to cluster i. Fill in the blank (RHS only)
						SCREAMING_SNAKE_CASE__									= data[cluster_assignment == i]
						# Compute the mean of the data points. Fill in the blank (RHS only)
						SCREAMING_SNAKE_CASE__									= member_data_points.mean(axis=0      )
						new_centroids.append(snake_case__      )
			SCREAMING_SNAKE_CASE__									= np.array(snake_case__      )

			return new_centroids

def 		A    (       snake_case__						,			snake_case__						,			snake_case__						,			snake_case__      ):




			'''simple docstring'''


			SCREAMING_SNAKE_CASE__									= 0.0
			for i in range(snake_case__      ):
						# Select all data points that belong to cluster i. Fill in the blank (RHS only)
						SCREAMING_SNAKE_CASE__									= data[cluster_assignment == i, :]

						if member_data_points.shape[0] > 0:  # check if i-th cluster is non-empty
									# Compute distances from centroid to data points (RHS only)
									SCREAMING_SNAKE_CASE__									= pairwise_distances(
									    snake_case__						,			[centroids[i]]						,			metric="""euclidean"""      )
									SCREAMING_SNAKE_CASE__									= distances**2
									heterogeneity += np.sum(snake_case__      )

			return heterogeneity

def 		A    (       snake_case__						,			snake_case__      ):




			'''simple docstring'''


			plt.figure(figsize=(7, 4)      )
			plt.plot(snake_case__						,			linewidth=4      )
			plt.xlabel("""# Iterations"""      )
			plt.ylabel("""Heterogeneity"""      )
			plt.title(f"""Heterogeneity of clustering over time, K={k:d}"""      )
			plt.rcParams.update({"""font.size""": 16}      )
			plt.show()

def 		A    (       snake_case__						,			snake_case__						,			snake_case__						,			snake_case__=5_00						,			snake_case__=None						,			snake_case__=False      ):




			'''simple docstring'''


			SCREAMING_SNAKE_CASE__									= initial_centroids[:]
			SCREAMING_SNAKE_CASE__									= None

			for itr in range(snake_case__      ):
						if verbose:
									print(snake_case__						,			end=""""""      )

						# 1. Make cluster assignments using nearest centroids
						SCREAMING_SNAKE_CASE__									= assign_clusters(snake_case__						,			snake_case__      )

						# 2. Compute a new centroid for each of the k clusters, averaging all data
						#    points assigned to that cluster.
						SCREAMING_SNAKE_CASE__									= revise_centroids(snake_case__						,			snake_case__						,			snake_case__      )

						# Check for convergence: if none of the assignments changed, stop
						if (
						    prev_cluster_assignment is not None
						    and (prev_cluster_assignment == cluster_assignment).all()
						):
									break

						# Print number of new assignments
						if prev_cluster_assignment is not None:
									SCREAMING_SNAKE_CASE__									= np.sum(prev_cluster_assignment != cluster_assignment      )
									if verbose:
												print(
												    f"""    {num_changed:5d} elements changed their cluster assignment."""      )

        # Record heterogeneity convergence metric
						if record_heterogeneity is not None:
									# YOUR CODE HERE
									SCREAMING_SNAKE_CASE__									= compute_heterogeneity(snake_case__						,			snake_case__						,			snake_case__						,			snake_case__      )
									record_heterogeneity.append(snake_case__      )

						SCREAMING_SNAKE_CASE__									= cluster_assignment[:]

			return centroids, cluster_assignment


# Mock test below
if False:  # change to true to run this test case.
						from sklearn import datasets as ds

						A_							:     Any     =	ds.load_iris()
						A_							:     Tuple     =	3
						A_							:     Optional[int]     =	[]
						A_							:     Union[str, Any]     =	get_initial_centroids(dataset["data"], k, seed=0)
						A_							:     int     =	kmeans(
						    dataset["data"],
						    k,
						    initial_centroids,
						    maxiter=400,
						    record_heterogeneity=heterogeneity,
						    verbose=True,
						)
						plot_heterogeneity(heterogeneity, k)

def 		A    (       snake_case__						,			snake_case__						,			snake_case__=None      ):




			'''simple docstring'''


			if fill_missing_report:
						SCREAMING_SNAKE_CASE__									= df.fillna(value=snake_case__      )
			SCREAMING_SNAKE_CASE__									= 1
			SCREAMING_SNAKE_CASE__									= df.select_dtypes(np.number      ).columns
			SCREAMING_SNAKE_CASE__									= (
			    df.groupby(["""Cluster"""]      )[  # construct report dataframe
			        numeric_cols
			    ]  # group by cluster number
			    .agg(
			        [
			            ("""sum""", np.sum),
			            ("""mean_with_zeros""", lambda snake_case__      : np.mean(np.nan_to_num(snake_case__      )      )),
			            ("""mean_without_zeros""", lambda snake_case__      : x.replace(0						,			np.NaN      ).mean()),
			            (
			                """mean_25-75""",
			                lambda snake_case__      : np.mean(
			                    np.nan_to_num(
			                        sorted(snake_case__      )[
			                            round(len(snake_case__      ) * 25 / 1_00      ) : round(len(snake_case__      ) * 75 / 1_00      )
			                        ]      )      ),
			            ),
			            ("""mean_with_na""", np.mean),
			            ("""min""", lambda snake_case__      : x.min()),
			            ("""5%""", lambda snake_case__      : x.quantile(0.05      )),
			            ("""25%""", lambda snake_case__      : x.quantile(0.25      )),
			            ("""50%""", lambda snake_case__      : x.quantile(0.50      )),
			            ("""75%""", lambda snake_case__      : x.quantile(0.75      )),
			            ("""95%""", lambda snake_case__      : x.quantile(0.95      )),
			            ("""max""", lambda snake_case__      : x.max()),
			            ("""count""", lambda snake_case__      : x.count()),
			            ("""stdev""", lambda snake_case__      : x.std()),
			            ("""mode""", lambda snake_case__      : x.mode()[0]),
			            ("""median""", lambda snake_case__      : x.median()),
			            ("""# > 0""", lambda snake_case__      : (x > 0).sum()),
			        ]      )
			    .T.reset_index()
			    .rename(index=snake_case__						,			columns={"""level_0""": """Features""", """level_1""": """Type"""}      )
			)  # rename columns
			# calculate the size of cluster(count of clientID's)
			SCREAMING_SNAKE_CASE__									= report[
			    (report["""Features"""] == """dummy""") & (report["""Type"""] == """count""")
			].copy()  # avoid SettingWithCopyWarning
			SCREAMING_SNAKE_CASE__									= (
			    """ClusterSize"""  # rename created cluster df to match report column names
			)
			SCREAMING_SNAKE_CASE__									= """# of Customers"""
			SCREAMING_SNAKE_CASE__									= pd.DataFrame(
			    clustersize.iloc[:, 2:].values
			    / clustersize.iloc[:, 2:].values.sum()      )
			SCREAMING_SNAKE_CASE__									= """% of Customers"""  # rename created cluster df to match report column names
			SCREAMING_SNAKE_CASE__									= """ClusterProportion"""
			SCREAMING_SNAKE_CASE__									= clusterproportion.columns.tolist()
			SCREAMING_SNAKE_CASE__									= cols[-2:] + cols[:-2]
			SCREAMING_SNAKE_CASE__									= clusterproportion[cols]  # rearrange columns to match report
			SCREAMING_SNAKE_CASE__									= report.columns
			SCREAMING_SNAKE_CASE__									= pd.DataFrame(
			    abs(
			        report[report["""Type"""] == """count"""].iloc[:, 2:].values
			        - clustersize.iloc[:, 2:].values      )      )  # generating df with count of nan values
			SCREAMING_SNAKE_CASE__									= 0
			SCREAMING_SNAKE_CASE__									= """# of nan"""
			SCREAMING_SNAKE_CASE__									= report[
			    report["""Type"""] == """count"""
			].Features.tolist()  # filling values in order to match report
			SCREAMING_SNAKE_CASE__									= a.columns.tolist()
			SCREAMING_SNAKE_CASE__									= cols[-2:] + cols[:-2]
			SCREAMING_SNAKE_CASE__									= a[cols]  # rearrange columns to match report
			SCREAMING_SNAKE_CASE__									= report.columns  # rename columns to match report
			SCREAMING_SNAKE_CASE__									= report.drop(
			    report[report.Type == """count"""].index      )  # drop count values except cluster size
			SCREAMING_SNAKE_CASE__									= pd.concat(
			    [report, a, clustersize, clusterproportion]						,			axis=0      )  # concat report with clustert size and nan values
			SCREAMING_SNAKE_CASE__									= report["""Features"""].isin(snake_case__      )
			SCREAMING_SNAKE_CASE__									= report.columns.tolist()
			SCREAMING_SNAKE_CASE__									= cols[0:2] + cols[-1:] + cols[2:-1]
			SCREAMING_SNAKE_CASE__									= report[cols]
			SCREAMING_SNAKE_CASE__									= {
			    """ClusterSize""": 9,
			    """ClusterProportion""": 8,
			    """mean_with_zeros""": 7,
			    """mean_with_na""": 6,
			    """max""": 5,
			    """50%""": 4,
			    """min""": 3,
			    """25%""": 2,
			    """75%""": 1,
			    """# of nan""": 0,
			    """# > 0""": -1,
			    """sum_with_na""": -2,
			}
			SCREAMING_SNAKE_CASE__									= (
			    report.assign(
			        Sortera=lambda snake_case__      : x.Type.map(snake_case__      )						,			Sortera=lambda snake_case__      : list(reversed(range(len(snake_case__      )      )      )      )						,			)
			    .sort_values(["""Sorter1""", """Mark""", """Sorter2"""]						,			ascending=snake_case__      )
			    .drop(["""Sorter1""", """Sorter2"""]						,			axis=1      )
			)
			SCREAMING_SNAKE_CASE__									= """"""
			SCREAMING_SNAKE_CASE__									= report.reset_index()
			SCREAMING_SNAKE_CASE__									= report.drop(columns=["""index"""]      )
			return report


if __name__ == "__main__":
						import doctest

						doctest.testmod()





"""simple docstring"""




import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
						from tokenizers import Tokenizer
						from tokenizers.models import WordPiece





class 				lowerCamelCase  (unittest.TestCase     ):

			def    SCREAMING_SNAKE_CASE			(   self			:  Union[str, Any]      ,			__UpperCAmelCase			:  str    )				->      int:
						SCREAMING_SNAKE_CASE__									= list(tokenizer_class.max_model_input_sizes.keys()    )
						for model_name in sa_models[:1]:
									SCREAMING_SNAKE_CASE__									= tokenizer_class.from_pretrained(__UpperCAmelCase    )
									self.assertIsNotNone(__UpperCAmelCase    )
									self.assertIsInstance(__UpperCAmelCase      ,			__UpperCAmelCase    )
									self.assertIsInstance(__UpperCAmelCase      ,			__UpperCAmelCase    )

									for special_tok in tokenizer.all_special_tokens:
												self.assertIsInstance(__UpperCAmelCase      ,			__UpperCAmelCase    )
												SCREAMING_SNAKE_CASE__									= tokenizer.convert_tokens_to_ids(__UpperCAmelCase    )
												self.assertIsInstance(__UpperCAmelCase      ,			__UpperCAmelCase    )

			def    SCREAMING_SNAKE_CASE			(   self			:  Dict      ,			__UpperCAmelCase			:  BatchEncoding      ,			__UpperCAmelCase			:  Optional[Callable] = None    )				->      Tuple:
						SCREAMING_SNAKE_CASE__									= pickle.dumps(__UpperCAmelCase    )
						self.assertIsNotNone(__UpperCAmelCase    )

						SCREAMING_SNAKE_CASE__									= pickle.loads(__UpperCAmelCase    )

						# Ensure is_fast is correctly restored
						self.assertEqual(be_restored.is_fast      ,			be_original.is_fast    )

						# Ensure encodings are potentially correctly restored
						if be_original.is_fast:
									self.assertIsNotNone(be_restored.encodings    )
						else:
									self.assertIsNone(be_restored.encodings    )

						# Ensure the keys are the same
						for original_v, restored_v in zip(be_original.values()      ,			be_restored.values()    ):
									if equal_op:
												self.assertTrue(equal_op(__UpperCAmelCase      ,			__UpperCAmelCase    )    )
									else:
												self.assertEqual(__UpperCAmelCase      ,			__UpperCAmelCase    )

			@slow
			def    SCREAMING_SNAKE_CASE			(   self			:  str    )				->      Optional[Any]:
						self.check_tokenizer_from_pretrained(__UpperCAmelCase    )

			def    SCREAMING_SNAKE_CASE			(   self			:  Tuple    )				->      Optional[int]:
						self.assertEqual(TensorType("""tf"""    )      ,			TensorType.TENSORFLOW    )
						self.assertEqual(TensorType("""pt"""    )      ,			TensorType.PYTORCH    )
						self.assertEqual(TensorType("""np"""    )      ,			TensorType.NUMPY    )

			@require_tokenizers
			def    SCREAMING_SNAKE_CASE			(   self			:  Dict    )				->      List[str]:
						import numpy as np

						SCREAMING_SNAKE_CASE__									= BertTokenizer.from_pretrained("""bert-base-cased"""    )
						SCREAMING_SNAKE_CASE__									= BertTokenizerFast.from_pretrained("""bert-base-cased"""    )

						# Python no tensor
						with self.subTest("""BatchEncoding (Python, return_tensors=None)"""    ):
									self.assert_dump_and_restore(tokenizer_p("""Small example to encode"""    )    )

						with self.subTest("""BatchEncoding (Python, return_tensors=NUMPY)"""    ):
									self.assert_dump_and_restore(
									    tokenizer_p("""Small example to encode"""      ,			return_tensors=TensorType.NUMPY    )      ,			np.array_equal    )

						with self.subTest("""BatchEncoding (Rust, return_tensors=None)"""    ):
									self.assert_dump_and_restore(tokenizer_r("""Small example to encode"""    )    )

						with self.subTest("""BatchEncoding (Rust, return_tensors=NUMPY)"""    ):
									self.assert_dump_and_restore(
									    tokenizer_r("""Small example to encode"""      ,			return_tensors=TensorType.NUMPY    )      ,			np.array_equal    )

			@require_tf
			@require_tokenizers
			def    SCREAMING_SNAKE_CASE			(   self			:  Dict    )				->      int:
						import tensorflow as tf

						def tf_array_equals(__UpperCAmelCase			:  Optional[int]      ,			__UpperCAmelCase			:  str    ):
									return tf.reduce_all(tf.equal(__UpperCAmelCase      ,			__UpperCAmelCase    )    )

						SCREAMING_SNAKE_CASE__									= BertTokenizer.from_pretrained("""bert-base-cased"""    )
						SCREAMING_SNAKE_CASE__									= BertTokenizerFast.from_pretrained("""bert-base-cased"""    )

						with self.subTest("""BatchEncoding (Python, return_tensors=TENSORFLOW)"""    ):
									self.assert_dump_and_restore(
									    tokenizer_p("""Small example to encode"""      ,			return_tensors=TensorType.TENSORFLOW    )      ,			__UpperCAmelCase    )

						with self.subTest("""BatchEncoding (Rust, return_tensors=TENSORFLOW)"""    ):
									self.assert_dump_and_restore(
									    tokenizer_r("""Small example to encode"""      ,			return_tensors=TensorType.TENSORFLOW    )      ,			__UpperCAmelCase    )

			@require_torch
			@require_tokenizers
			def    SCREAMING_SNAKE_CASE			(   self			:  Optional[int]    )				->      Optional[int]:
						import torch

						SCREAMING_SNAKE_CASE__									= BertTokenizer.from_pretrained("""bert-base-cased"""    )
						SCREAMING_SNAKE_CASE__									= BertTokenizerFast.from_pretrained("""bert-base-cased"""    )

						with self.subTest("""BatchEncoding (Python, return_tensors=PYTORCH)"""    ):
									self.assert_dump_and_restore(
									    tokenizer_p("""Small example to encode"""      ,			return_tensors=TensorType.PYTORCH    )      ,			torch.equal    )

						with self.subTest("""BatchEncoding (Rust, return_tensors=PYTORCH)"""    ):
									self.assert_dump_and_restore(
									    tokenizer_r("""Small example to encode"""      ,			return_tensors=TensorType.PYTORCH    )      ,			torch.equal    )

			@require_tokenizers
			def    SCREAMING_SNAKE_CASE			(   self			:  Any    )				->      int:
						SCREAMING_SNAKE_CASE__									= BertTokenizer.from_pretrained("""bert-base-cased"""    )
						SCREAMING_SNAKE_CASE__									= BertTokenizerFast.from_pretrained("""bert-base-cased"""    )

						with self.subTest("""Python Tokenizer"""    ):
									self.assertFalse(tokenizer_p("""Small example to_encode"""    ).is_fast    )

						with self.subTest("""Rust Tokenizer"""    ):
									self.assertTrue(tokenizer_r("""Small example to_encode"""    ).is_fast    )

			@require_tokenizers
			def    SCREAMING_SNAKE_CASE			(   self			:  Optional[int]    )				->      Optional[Any]:
						SCREAMING_SNAKE_CASE__									= BertTokenizerFast.from_pretrained("""bert-base-cased"""    )
						SCREAMING_SNAKE_CASE__									= tokenizer_r(["""Test""", """\xad""", """test"""]      ,			is_split_into_words=__UpperCAmelCase    )

						self.assertEqual(encoded.word_to_tokens(0    )      ,			TokenSpan(start=1      ,			end=2    )    )
						self.assertEqual(encoded.word_to_tokens(1    )      ,			__UpperCAmelCase    )
						self.assertEqual(encoded.word_to_tokens(2    )      ,			TokenSpan(start=2      ,			end=3    )    )

			def    SCREAMING_SNAKE_CASE			(   self			:  Optional[Any]    )				->      Optional[int]:
						SCREAMING_SNAKE_CASE__									= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}    )
						SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""np"""    )
						self.assertEqual(tensor_batch["""inputs"""].shape      ,			(2, 3)    )
						self.assertEqual(tensor_batch["""labels"""].shape      ,			(2,)    )
						# test converting the converted
						with CaptureStderr() as cs:
									SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""np"""    )
						self.assertFalse(len(cs.err    )      ,			msg=F"""should have no warning, but got {cs.err}"""    )

						SCREAMING_SNAKE_CASE__									= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}    )
						SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""np"""      ,			prepend_batch_axis=__UpperCAmelCase    )
						self.assertEqual(tensor_batch["""inputs"""].shape      ,			(1, 3)    )
						self.assertEqual(tensor_batch["""labels"""].shape      ,			(1,)    )

			@require_torch
			def    SCREAMING_SNAKE_CASE			(   self			:  Dict    )				->      Optional[Any]:
						SCREAMING_SNAKE_CASE__									= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}    )
						SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""pt"""    )
						self.assertEqual(tensor_batch["""inputs"""].shape      ,			(2, 3)    )
						self.assertEqual(tensor_batch["""labels"""].shape      ,			(2,)    )
						# test converting the converted
						with CaptureStderr() as cs:
									SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""pt"""    )
						self.assertFalse(len(cs.err    )      ,			msg=F"""should have no warning, but got {cs.err}"""    )

						SCREAMING_SNAKE_CASE__									= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}    )
						SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""pt"""      ,			prepend_batch_axis=__UpperCAmelCase    )
						self.assertEqual(tensor_batch["""inputs"""].shape      ,			(1, 3)    )
						self.assertEqual(tensor_batch["""labels"""].shape      ,			(1,)    )

			@require_tf
			def    SCREAMING_SNAKE_CASE			(   self			:  Any    )				->      int:
						SCREAMING_SNAKE_CASE__									= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}    )
						SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""tf"""    )
						self.assertEqual(tensor_batch["""inputs"""].shape      ,			(2, 3)    )
						self.assertEqual(tensor_batch["""labels"""].shape      ,			(2,)    )
						# test converting the converted
						with CaptureStderr() as cs:
									SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""tf"""    )
						self.assertFalse(len(cs.err    )      ,			msg=F"""should have no warning, but got {cs.err}"""    )

						SCREAMING_SNAKE_CASE__									= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}    )
						SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""tf"""      ,			prepend_batch_axis=__UpperCAmelCase    )
						self.assertEqual(tensor_batch["""inputs"""].shape      ,			(1, 3)    )
						self.assertEqual(tensor_batch["""labels"""].shape      ,			(1,)    )

			@require_flax
			def    SCREAMING_SNAKE_CASE			(   self			:  int    )				->      Tuple:
						SCREAMING_SNAKE_CASE__									= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}    )
						SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""jax"""    )
						self.assertEqual(tensor_batch["""inputs"""].shape      ,			(2, 3)    )
						self.assertEqual(tensor_batch["""labels"""].shape      ,			(2,)    )
						# test converting the converted
						with CaptureStderr() as cs:
									SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""jax"""    )
						self.assertFalse(len(cs.err    )      ,			msg=F"""should have no warning, but got {cs.err}"""    )

						SCREAMING_SNAKE_CASE__									= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}    )
						SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""jax"""      ,			prepend_batch_axis=__UpperCAmelCase    )
						self.assertEqual(tensor_batch["""inputs"""].shape      ,			(1, 3)    )
						self.assertEqual(tensor_batch["""labels"""].shape      ,			(1,)    )

			def    SCREAMING_SNAKE_CASE			(   self			:  List[Any]    )				->      List[Any]:
						SCREAMING_SNAKE_CASE__									= [{"""input_ids""": np.array([0, 1, 2]    )}, {"""input_ids""": np.array([0, 1, 2, 3]    )}]
						SCREAMING_SNAKE_CASE__									= BertTokenizer.from_pretrained("""bert-base-cased"""    )

						SCREAMING_SNAKE_CASE__									= tokenizer.pad(__UpperCAmelCase      ,			padding=__UpperCAmelCase    )
						self.assertTrue(isinstance(batch["""input_ids"""]      ,			np.ndarray    )    )
						self.assertEqual(batch["""input_ids"""].tolist()      ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )
						SCREAMING_SNAKE_CASE__									= tokenizer.pad(__UpperCAmelCase      ,			padding=__UpperCAmelCase      ,			return_tensors="""np"""    )
						self.assertTrue(isinstance(batch["""input_ids"""]      ,			np.ndarray    )    )
						self.assertEqual(batch["""input_ids"""].tolist()      ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )

			@require_torch
			def    SCREAMING_SNAKE_CASE			(   self			:  Dict    )				->      Optional[int]:
						import torch

						SCREAMING_SNAKE_CASE__									= [{"""input_ids""": torch.tensor([0, 1, 2]    )}, {"""input_ids""": torch.tensor([0, 1, 2, 3]    )}]
						SCREAMING_SNAKE_CASE__									= BertTokenizer.from_pretrained("""bert-base-cased"""    )

						SCREAMING_SNAKE_CASE__									= tokenizer.pad(__UpperCAmelCase      ,			padding=__UpperCAmelCase    )
						self.assertTrue(isinstance(batch["""input_ids"""]      ,			torch.Tensor    )    )
						self.assertEqual(batch["""input_ids"""].tolist()      ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )
						SCREAMING_SNAKE_CASE__									= tokenizer.pad(__UpperCAmelCase      ,			padding=__UpperCAmelCase      ,			return_tensors="""pt"""    )
						self.assertTrue(isinstance(batch["""input_ids"""]      ,			torch.Tensor    )    )
						self.assertEqual(batch["""input_ids"""].tolist()      ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )

			@require_tf
			def    SCREAMING_SNAKE_CASE			(   self			:  List[Any]    )				->      Dict:
						import tensorflow as tf

						SCREAMING_SNAKE_CASE__									= [{"""input_ids""": tf.constant([0, 1, 2]    )}, {"""input_ids""": tf.constant([0, 1, 2, 3]    )}]
						SCREAMING_SNAKE_CASE__									= BertTokenizer.from_pretrained("""bert-base-cased"""    )

						SCREAMING_SNAKE_CASE__									= tokenizer.pad(__UpperCAmelCase      ,			padding=__UpperCAmelCase    )
						self.assertTrue(isinstance(batch["""input_ids"""]      ,			tf.Tensor    )    )
						self.assertEqual(batch["""input_ids"""].numpy().tolist()      ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )
						SCREAMING_SNAKE_CASE__									= tokenizer.pad(__UpperCAmelCase      ,			padding=__UpperCAmelCase      ,			return_tensors="""tf"""    )
						self.assertTrue(isinstance(batch["""input_ids"""]      ,			tf.Tensor    )    )
						self.assertEqual(batch["""input_ids"""].numpy().tolist()      ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )

			@require_tokenizers
			def    SCREAMING_SNAKE_CASE			(   self			:  Optional[Any]    )				->      Optional[int]:
						SCREAMING_SNAKE_CASE__									= Tokenizer(WordPiece(unk_token="""[UNK]"""    )    )
						PreTrainedTokenizerFast(tokenizer_object=__UpperCAmelCase    )





			@require_tokenizers
			def    SCREAMING_SNAKE_CASE			(   self			:  str    )				->      int:
						SCREAMING_SNAKE_CASE__									= Tokenizer(WordPiece(unk_token="""[UNK]"""    )    )
						with tempfile.TemporaryDirectory() as tmpdirname:
									bert_tokenizer.save(os.path.join(__UpperCAmelCase      ,			"""tokenizer.json"""    )    )
									PreTrainedTokenizerFast(tokenizer_file=os.path.join(__UpperCAmelCase      ,			"""tokenizer.json"""    )    )


