
import warnings

import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from sklearn.metrics import pairwise_distances

warnings.filterwarnings("ignore")

lowercase__							:							int													=     "K-MEANS-CLUST/ "




def 		A_	(							snake_case							:				List[Any]	, snake_case							:				Dict	, snake_case							:				Optional[int]=None	) ->      int:



						'''simple docstring'''

						if seed is not None:  # useful for obtaining consistent results
												np.random.seed(snake_case	)
						__UpperCamelCase       					=     data.shape[0]  # number of data points

						# Pick K indices from range [0, N).
						__UpperCamelCase       					=     np.random.randint(0	, snake_case	, snake_case	)

						# Keep centroids as dense format, as many entries will be nonzero due to averaging.
						# As long as at least one document in a cluster contains a word,
						# it will carry a nonzero weight in the TF-IDF vector of the centroid.
						__UpperCamelCase       					=     data[rand_indices, :]

						return centroids




def 		A_	(							snake_case							:				List[Any]	, snake_case							:				Optional[int]	) ->      Tuple:



						'''simple docstring'''

						return pairwise_distances(snake_case	, snake_case	, metric='''euclidean'''	)




def 		A_	(							snake_case							:				Tuple	, snake_case							:				Dict	) ->      List[str]:



						'''simple docstring'''

						__UpperCamelCase       					=     centroid_pairwise_dist(snake_case	, snake_case	)

						# Compute cluster assignments for each data point:
						# Fill in the blank (RHS only)
						__UpperCamelCase       					=     np.argmin(snake_case	, axis=1	)

						return cluster_assignment




def 		A_	(							snake_case							:				Dict	, snake_case							:				Dict	, snake_case							:				List[Any]	) ->      str:



						'''simple docstring'''

						__UpperCamelCase       					=     []
						for i in range(snake_case	):
												# Select all data points that belong to cluster i. Fill in the blank (RHS only)
												__UpperCamelCase       					=     data[cluster_assignment == i]
												# Compute the mean of the data points. Fill in the blank (RHS only)
												__UpperCamelCase       					=     member_data_points.mean(axis=0	)
												new_centroids.append(snake_case	)
						__UpperCamelCase       					=     np.array(snake_case	)

						return new_centroids




def 		A_	(							snake_case							:				List[str]	, snake_case							:				List[Any]	, snake_case							:				str	, snake_case							:				Union[str, Any]	) ->      Dict:



						'''simple docstring'''

						__UpperCamelCase       					=     0.0
						for i in range(snake_case	):
												# Select all data points that belong to cluster i. Fill in the blank (RHS only)
												__UpperCamelCase       					=     data[cluster_assignment == i, :]

												if member_data_points.shape[0] > 0:  # check if i-th cluster is non-empty
																		# Compute distances from centroid to data points (RHS only)
																		__UpperCamelCase       					=     pairwise_distances(
																		    snake_case	, [centroids[i]]	, metric='''euclidean'''	)
																		__UpperCamelCase       					=     distances**2
																		heterogeneity += np.sum(snake_case	)

						return heterogeneity




def 		A_	(							snake_case							:				Optional[int]	, snake_case							:				str	) ->      Dict:



						'''simple docstring'''

						plt.figure(figsize=(7, 4)	)
						plt.plot(snake_case	, linewidth=4	)
						plt.xlabel('''# Iterations'''	)
						plt.ylabel('''Heterogeneity'''	)
						plt.title(f"Heterogeneity of clustering over time, K={k:d}"	)
						plt.rcParams.update({'''font.size''': 16}	)
						plt.show()




def 		A_	(							snake_case							:				Optional[int]	, snake_case							:				Optional[int]	, snake_case							:				Tuple	, snake_case							:				List[str]=500	, snake_case							:				Dict=None	, snake_case							:				Optional[int]=False	) ->      Union[str, Any]:



						'''simple docstring'''

						__UpperCamelCase       					=     initial_centroids[:]
						__UpperCamelCase       					=     None

						for itr in range(snake_case	):
												if verbose:
																		print(snake_case	, end=''''''	)

												# 1. Make cluster assignments using nearest centroids
												__UpperCamelCase       					=     assign_clusters(snake_case	, snake_case	)

												# 2. Compute a new centroid for each of the k clusters, averaging all data
												#    points assigned to that cluster.
												__UpperCamelCase       					=     revise_centroids(snake_case	, snake_case	, snake_case	)

												# Check for convergence: if none of the assignments changed, stop
												if (
												    prev_cluster_assignment is not None
												    and (prev_cluster_assignment == cluster_assignment).all()
												):
																		break

												# Print number of new assignments
												if prev_cluster_assignment is not None:
																		__UpperCamelCase       					=     np.sum(prev_cluster_assignment != cluster_assignment	)
																		if verbose:
																								print(
																								    f"    {num_changed:5d} elements changed their cluster assignment."	)

        # Record heterogeneity convergence metric
												if record_heterogeneity is not None:
																		# YOUR CODE HERE
																		__UpperCamelCase       					=     compute_heterogeneity(snake_case	, snake_case	, snake_case	, snake_case	)
																		record_heterogeneity.append(snake_case	)

												__UpperCamelCase       					=     cluster_assignment[:]

						return centroids, cluster_assignment


# Mock test below
if False:  # change to true to run this test case.
				from sklearn import datasets as ds

				lowercase__							:							Any													=     ds.load_iris()
				lowercase__							:							Tuple													=     3
				lowercase__							:							Optional[int]													=     []
				lowercase__							:							Union[str, Any]													=     get_initial_centroids(dataset["data"], k, seed=0)
				lowercase__						,					lowercase__							:							int													=     kmeans(
				    dataset["data"],
				    k,
				    initial_centroids,
				    maxiter=4_0_0,
				    record_heterogeneity=heterogeneity,
				    verbose=True,
				)
				plot_heterogeneity(heterogeneity, k)




def 		A_	(							snake_case							:				pd.DataFrame	, snake_case							:				np.ndarray	, snake_case							:				Optional[Any]=None	) ->      pd.DataFrame:



						'''simple docstring'''

						if fill_missing_report:
												__UpperCamelCase       					=     df.fillna(value=snake_case	)
						__UpperCamelCase       					=     1
						__UpperCamelCase       					=     df.select_dtypes(np.number	).columns
						__UpperCamelCase       					=     (
						    df.groupby(['''Cluster''']	)[  # construct report dataframe
						        numeric_cols
						    ]  # group by cluster number
						    .agg(
						        [
						            ('''sum''', np.sum),
						            ('''mean_with_zeros''', lambda snake_case	: np.mean(np.nan_to_num(snake_case	)	)),
						            ('''mean_without_zeros''', lambda snake_case	: x.replace(0	, np.NaN	).mean()),
						            (
						                '''mean_25-75''',
						                lambda snake_case	: np.mean(
						                    np.nan_to_num(
						                        sorted(snake_case	)[
						                            round(len(snake_case	) * 25 / 100	) : round(len(snake_case	) * 75 / 100	)
						                        ]	)	),
						            ),
						            ('''mean_with_na''', np.mean),
						            ('''min''', lambda snake_case	: x.min()),
						            ('''5%''', lambda snake_case	: x.quantile(0.05	)),
						            ('''25%''', lambda snake_case	: x.quantile(0.25	)),
						            ('''50%''', lambda snake_case	: x.quantile(0.50	)),
						            ('''75%''', lambda snake_case	: x.quantile(0.75	)),
						            ('''95%''', lambda snake_case	: x.quantile(0.95	)),
						            ('''max''', lambda snake_case	: x.max()),
						            ('''count''', lambda snake_case	: x.count()),
						            ('''stdev''', lambda snake_case	: x.std()),
						            ('''mode''', lambda snake_case	: x.mode()[0]),
						            ('''median''', lambda snake_case	: x.median()),
						            ('''# > 0''', lambda snake_case	: (x > 0).sum()),
						        ]	)
						    .T.reset_index()
						    .rename(index=snake_case	, columns={'''level_0''': '''Features''', '''level_1''': '''Type'''}	)
						)  # rename columns
						# calculate the size of cluster(count of clientID's)
						__UpperCamelCase       					=     report[
						    (report['''Features'''] == '''dummy''') & (report['''Type'''] == '''count''')
						].copy()  # avoid SettingWithCopyWarning
						__UpperCamelCase       					=     (
						    '''ClusterSize'''  # rename created cluster df to match report column names
						)
						__UpperCamelCase       					=     '''# of Customers'''
						__UpperCamelCase       					=     pd.DataFrame(
						    clustersize.iloc[:, 2:].values
						    / clustersize.iloc[:, 2:].values.sum()	)
						__UpperCamelCase       					=     '''% of Customers'''  # rename created cluster df to match report column names
						__UpperCamelCase       					=     '''ClusterProportion'''
						__UpperCamelCase       					=     clusterproportion.columns.tolist()
						__UpperCamelCase       					=     cols[-2:] + cols[:-2]
						__UpperCamelCase       					=     clusterproportion[cols]  # rearrange columns to match report
						__UpperCamelCase       					=     report.columns
						__UpperCamelCase       					=     pd.DataFrame(
						    abs(
						        report[report['''Type'''] == '''count'''].iloc[:, 2:].values
						        - clustersize.iloc[:, 2:].values	)	)  # generating df with count of nan values
						__UpperCamelCase       					=     0
						__UpperCamelCase       					=     '''# of nan'''
						__UpperCamelCase       					=     report[
						    report['''Type'''] == '''count'''
						].Features.tolist()  # filling values in order to match report
						__UpperCamelCase       					=     a.columns.tolist()
						__UpperCamelCase       					=     cols[-2:] + cols[:-2]
						__UpperCamelCase       					=     a[cols]  # rearrange columns to match report
						__UpperCamelCase       					=     report.columns  # rename columns to match report
						__UpperCamelCase       					=     report.drop(
						    report[report.Type == '''count'''].index	)  # drop count values except cluster size
						__UpperCamelCase       					=     pd.concat(
						    [report, a, clustersize, clusterproportion]	, axis=0	)  # concat report with clustert size and nan values
						__UpperCamelCase       					=     report['''Features'''].isin(snake_case	)
						__UpperCamelCase       					=     report.columns.tolist()
						__UpperCamelCase       					=     cols[0:2] + cols[-1:] + cols[2:-1]
						__UpperCamelCase       					=     report[cols]
						__UpperCamelCase       					=     {
						    '''ClusterSize''': 9,
						    '''ClusterProportion''': 8,
						    '''mean_with_zeros''': 7,
						    '''mean_with_na''': 6,
						    '''max''': 5,
						    '''50%''': 4,
						    '''min''': 3,
						    '''25%''': 2,
						    '''75%''': 1,
						    '''# of nan''': 0,
						    '''# > 0''': -1,
						    '''sum_with_na''': -2,
						}
						__UpperCamelCase       					=     (
						    report.assign(
						        Sortera=lambda snake_case	: x.Type.map(snake_case	)	, Sortera=lambda snake_case	: list(reversed(range(len(snake_case	)	)	)	)	, )
						    .sort_values(['''Sorter1''', '''Mark''', '''Sorter2''']	, ascending=snake_case	)
						    .drop(['''Sorter1''', '''Sorter2''']	, axis=1	)
						)
						__UpperCamelCase       					=     ''''''
						__UpperCamelCase       					=     report.reset_index()
						__UpperCamelCase       					=     report.drop(columns=['''index''']	)
						return report


if __name__ == "__main__":
				import doctest

				doctest.testmod()






import json
import os
import shutil
import tempfile
import unittest

from transformers import is_speech_available, is_torch_available
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_torch
from transformers.utils import FEATURE_EXTRACTOR_NAME


if is_speech_available() and is_torch_available():
				from transformers import SpeechTaFeatureExtractor, SpeechTaProcessor

				from .test_feature_extraction_speechta import floats_list


lowercase__							:							List[Any]													=     get_tests_dir("fixtures/test_sentencepiece_bpe_char.model")






@require_torch
class     SCREAMING_SNAKE_CASE__      (							unittest.TestCase      ):

							"""simple docstring"""






							def       A__						(					self     )->				List[Any]:





													'''simple docstring'''






													__UpperCamelCase       					=     tempfile.mkdtemp()

													__UpperCamelCase       					=     SpeechTaTokenizer(SCREAMING_SNAKE_CASE_     )
													tokenizer.save_pretrained(self.tmpdirname     )

													__UpperCamelCase       					=     {
													    '''feature_size''': 1,
													    '''padding_value''': 0.0,
													    '''sampling_rate''': 16000,
													    '''do_normalize''': False,
													    '''num_mel_bins''': 80,
													    '''hop_length''': 16,
													    '''win_length''': 64,
													    '''win_function''': '''hann_window''',
													    '''fmin''': 80,
													    '''fmax''': 7600,
													    '''mel_floor''': 1E-10,
													    '''reduction_factor''': 2,
													    '''return_attention_mask''': True,
													}

													__UpperCamelCase       					=     os.path.join(self.tmpdirname       ,		SCREAMING_SNAKE_CASE_     )
													with open(self.feature_extraction_file       ,		'''w'''       ,		encoding='''utf-8'''     ) as fp:
																			fp.write(json.dumps(SCREAMING_SNAKE_CASE_     ) + '''\n'''     )



							def       A__						(					self       ,		**SCREAMING_SNAKE_CASE_     )->				Union[str, Any]:





													'''simple docstring'''






													return SpeechTaTokenizer.from_pretrained(self.tmpdirname       ,		**SCREAMING_SNAKE_CASE_     )



							def       A__						(					self       ,		**SCREAMING_SNAKE_CASE_     )->				Union[str, Any]:





													'''simple docstring'''






													return SpeechTaFeatureExtractor.from_pretrained(self.tmpdirname       ,		**SCREAMING_SNAKE_CASE_     )



							def       A__						(					self     )->				str:





													'''simple docstring'''






													shutil.rmtree(self.tmpdirname     )



							def       A__						(					self     )->				Optional[int]:





													'''simple docstring'''






													__UpperCamelCase       					=     self.get_tokenizer()
													__UpperCamelCase       					=     self.get_feature_extractor()

													__UpperCamelCase       					=     SpeechTaProcessor(tokenizer=SCREAMING_SNAKE_CASE_       ,		feature_extractor=SCREAMING_SNAKE_CASE_     )

													processor.save_pretrained(self.tmpdirname     )
													__UpperCamelCase       					=     SpeechTaProcessor.from_pretrained(self.tmpdirname     )

													self.assertEqual(processor.tokenizer.get_vocab()       ,		tokenizer.get_vocab()     )
													self.assertIsInstance(processor.tokenizer       ,		SCREAMING_SNAKE_CASE_     )

													self.assertEqual(processor.feature_extractor.to_json_string()       ,		feature_extractor.to_json_string()     )
													self.assertIsInstance(processor.feature_extractor       ,		SCREAMING_SNAKE_CASE_     )



							def       A__						(					self     )->				List[str]:





													'''simple docstring'''






													__UpperCamelCase       					=     SpeechTaProcessor(tokenizer=self.get_tokenizer()       ,		feature_extractor=self.get_feature_extractor()     )
													processor.save_pretrained(self.tmpdirname     )

													__UpperCamelCase       					=     self.get_tokenizer(bos_token='''(BOS)'''       ,		eos_token='''(EOS)'''     )
													__UpperCamelCase       					=     self.get_feature_extractor(do_normalize=SCREAMING_SNAKE_CASE_       ,		padding_value=1.0     )

													__UpperCamelCase       					=     SpeechTaProcessor.from_pretrained(
													    self.tmpdirname       ,		bos_token='''(BOS)'''       ,		eos_token='''(EOS)'''       ,		do_normalize=SCREAMING_SNAKE_CASE_       ,		padding_value=1.0     )

													self.assertEqual(processor.tokenizer.get_vocab()       ,		tokenizer_add_kwargs.get_vocab()     )
													self.assertIsInstance(processor.tokenizer       ,		SCREAMING_SNAKE_CASE_     )

													self.assertEqual(processor.feature_extractor.to_json_string()       ,		feature_extractor_add_kwargs.to_json_string()     )
													self.assertIsInstance(processor.feature_extractor       ,		SCREAMING_SNAKE_CASE_     )



							def       A__						(					self     )->				Dict:





													'''simple docstring'''






													__UpperCamelCase       					=     self.get_feature_extractor()
													__UpperCamelCase       					=     self.get_tokenizer()

													__UpperCamelCase       					=     SpeechTaProcessor(tokenizer=SCREAMING_SNAKE_CASE_       ,		feature_extractor=SCREAMING_SNAKE_CASE_     )

													__UpperCamelCase       					=     floats_list((3, 1000)     )

													__UpperCamelCase       					=     feature_extractor(audio=SCREAMING_SNAKE_CASE_       ,		return_tensors='''np'''     )
													__UpperCamelCase       					=     processor(audio=SCREAMING_SNAKE_CASE_       ,		return_tensors='''np'''     )

													for key in input_feat_extract.keys():
																			self.assertAlmostEqual(input_feat_extract[key].sum()       ,		input_processor[key].sum()       ,		delta=1E-2     )



							def       A__						(					self     )->				Dict:





													'''simple docstring'''






													__UpperCamelCase       					=     self.get_feature_extractor()
													__UpperCamelCase       					=     self.get_tokenizer()

													__UpperCamelCase       					=     SpeechTaProcessor(tokenizer=SCREAMING_SNAKE_CASE_       ,		feature_extractor=SCREAMING_SNAKE_CASE_     )

													__UpperCamelCase       					=     floats_list((3, 1000)     )

													__UpperCamelCase       					=     feature_extractor(audio_target=SCREAMING_SNAKE_CASE_       ,		return_tensors='''np'''     )
													__UpperCamelCase       					=     processor(audio_target=SCREAMING_SNAKE_CASE_       ,		return_tensors='''np'''     )

													for key in input_feat_extract.keys():
																			self.assertAlmostEqual(input_feat_extract[key].sum()       ,		input_processor[key].sum()       ,		delta=1E-2     )



							def       A__						(					self     )->				int:





													'''simple docstring'''






													__UpperCamelCase       					=     self.get_feature_extractor()
													__UpperCamelCase       					=     self.get_tokenizer()

													__UpperCamelCase       					=     SpeechTaProcessor(tokenizer=SCREAMING_SNAKE_CASE_       ,		feature_extractor=SCREAMING_SNAKE_CASE_     )

													__UpperCamelCase       					=     '''This is a test string'''

													__UpperCamelCase       					=     processor(text=SCREAMING_SNAKE_CASE_     )
													__UpperCamelCase       					=     tokenizer(SCREAMING_SNAKE_CASE_     )

													for key in encoded_tok.keys():
																			self.assertListEqual(encoded_tok[key]       ,		encoded_processor[key]     )



							def       A__						(					self     )->				int:





													'''simple docstring'''






													__UpperCamelCase       					=     self.get_feature_extractor()
													__UpperCamelCase       					=     self.get_tokenizer()

													__UpperCamelCase       					=     SpeechTaProcessor(tokenizer=SCREAMING_SNAKE_CASE_       ,		feature_extractor=SCREAMING_SNAKE_CASE_     )

													__UpperCamelCase       					=     '''This is a test string'''

													__UpperCamelCase       					=     processor(text_target=SCREAMING_SNAKE_CASE_     )
													__UpperCamelCase       					=     tokenizer(SCREAMING_SNAKE_CASE_     )

													for key in encoded_tok.keys():
																			self.assertListEqual(encoded_tok[key]       ,		encoded_processor[key]     )



							def       A__						(					self     )->				Dict:





													'''simple docstring'''






													__UpperCamelCase       					=     self.get_feature_extractor()
													__UpperCamelCase       					=     self.get_tokenizer()

													__UpperCamelCase       					=     SpeechTaProcessor(tokenizer=SCREAMING_SNAKE_CASE_       ,		feature_extractor=SCREAMING_SNAKE_CASE_     )

													__UpperCamelCase       					=     [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

													__UpperCamelCase       					=     processor.batch_decode(SCREAMING_SNAKE_CASE_     )
													__UpperCamelCase       					=     tokenizer.batch_decode(SCREAMING_SNAKE_CASE_     )

													self.assertListEqual(SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_     )



							def       A__						(					self     )->				List[str]:





													'''simple docstring'''






													__UpperCamelCase       					=     self.get_feature_extractor()
													__UpperCamelCase       					=     self.get_tokenizer()

													__UpperCamelCase       					=     SpeechTaProcessor(tokenizer=SCREAMING_SNAKE_CASE_       ,		feature_extractor=SCREAMING_SNAKE_CASE_     )

													self.assertListEqual(
													    processor.model_input_names       ,		feature_extractor.model_input_names       ,		msg='''`processor` and `feature_extractor` model input names do not match'''       ,		)




