
import warnings

import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from sklearn.metrics import pairwise_distances

warnings.filterwarnings("ignore")

_A       =					"K-MEANS-CLUST/ "






def        lowerCamelCase__    (    __lowerCAmelCase  :  Union[str, Any]   ,							__lowerCAmelCase  :  str   ,							__lowerCAmelCase  :  Optional[Any]=None    ):




			"""simple docstring"""


			if seed is not None:  # useful for obtaining consistent results
						np.random.seed(__lowerCAmelCase    )
			lowerCAmelCase_      =     data.shape[0]  # number of data points

			# Pick K indices from range [0, N).
			lowerCAmelCase_      =     np.random.randint(0   ,							__lowerCAmelCase   ,							__lowerCAmelCase    )

			# Keep centroids as dense format, as many entries will be nonzero due to averaging.
			# As long as at least one document in a cluster contains a word,
			# it will carry a nonzero weight in the TF-IDF vector of the centroid.
			lowerCAmelCase_      =     data[rand_indices, :]

			return centroids






def        lowerCamelCase__    (    __lowerCAmelCase  :  Any   ,							__lowerCAmelCase  :  List[Any]    ):




			"""simple docstring"""


			return pairwise_distances(__lowerCAmelCase   ,							__lowerCAmelCase   ,							metric="euclidean"    )






def        lowerCamelCase__    (    __lowerCAmelCase  :  Union[str, Any]   ,							__lowerCAmelCase  :  Optional[int]    ):




			"""simple docstring"""


			lowerCAmelCase_      =     centroid_pairwise_dist(__lowerCAmelCase   ,							__lowerCAmelCase    )

			# Compute cluster assignments for each data point:
			# Fill in the blank (RHS only)
			lowerCAmelCase_      =     np.argmin(__lowerCAmelCase   ,							axis=1    )

			return cluster_assignment






def        lowerCamelCase__    (    __lowerCAmelCase  :  List[str]   ,							__lowerCAmelCase  :  List[Any]   ,							__lowerCAmelCase  :  str    ):




			"""simple docstring"""


			lowerCAmelCase_      =     []
			for i in range(__lowerCAmelCase    ):
						# Select all data points that belong to cluster i. Fill in the blank (RHS only)
						lowerCAmelCase_      =     data[cluster_assignment == i]
						# Compute the mean of the data points. Fill in the blank (RHS only)
						lowerCAmelCase_      =     member_data_points.mean(axis=0    )
						new_centroids.append(__lowerCAmelCase    )
			lowerCAmelCase_      =     np.array(__lowerCAmelCase    )

			return new_centroids






def        lowerCamelCase__    (    __lowerCAmelCase  :  Tuple   ,							__lowerCAmelCase  :  int   ,							__lowerCAmelCase  :  Union[str, Any]   ,							__lowerCAmelCase  :  List[str]    ):




			"""simple docstring"""


			lowerCAmelCase_      =     0.0
			for i in range(__lowerCAmelCase    ):
						# Select all data points that belong to cluster i. Fill in the blank (RHS only)
						lowerCAmelCase_      =     data[cluster_assignment == i, :]

						if member_data_points.shape[0] > 0:  # check if i-th cluster is non-empty
									# Compute distances from centroid to data points (RHS only)
									lowerCAmelCase_      =     pairwise_distances(
									    __lowerCAmelCase   ,							[centroids[i]]   ,							metric="euclidean"    )
									lowerCAmelCase_      =     distances**2
									heterogeneity += np.sum(__lowerCAmelCase    )

			return heterogeneity






def        lowerCamelCase__    (    __lowerCAmelCase  :  str   ,							__lowerCAmelCase  :  Any    ):




			"""simple docstring"""


			plt.figure(figsize=(7, 4)    )
			plt.plot(__lowerCAmelCase   ,							linewidth=4    )
			plt.xlabel("# Iterations"    )
			plt.ylabel("Heterogeneity"    )
			plt.title(F"""Heterogeneity of clustering over time, K={k:d}"""    )
			plt.rcParams.update({"font.size": 16}    )
			plt.show()






def        lowerCamelCase__    (    __lowerCAmelCase  :  Optional[int]   ,							__lowerCAmelCase  :  Optional[int]   ,							__lowerCAmelCase  :  int   ,							__lowerCAmelCase  :  Any=500   ,							__lowerCAmelCase  :  Dict=None   ,							__lowerCAmelCase  :  Any=False    ):




			"""simple docstring"""


			lowerCAmelCase_      =     initial_centroids[:]
			lowerCAmelCase_      =     None

			for itr in range(__lowerCAmelCase    ):
						if verbose:
									print(__lowerCAmelCase   ,							end=""    )

						# 1. Make cluster assignments using nearest centroids
						lowerCAmelCase_      =     assign_clusters(__lowerCAmelCase   ,							__lowerCAmelCase    )

						# 2. Compute a new centroid for each of the k clusters, averaging all data
						#    points assigned to that cluster.
						lowerCAmelCase_      =     revise_centroids(__lowerCAmelCase   ,							__lowerCAmelCase   ,							__lowerCAmelCase    )

						# Check for convergence: if none of the assignments changed, stop
						if (
						    prev_cluster_assignment is not None
						    and (prev_cluster_assignment == cluster_assignment).all()
						):
									break

						# Print number of new assignments
						if prev_cluster_assignment is not None:
									lowerCAmelCase_      =     np.sum(prev_cluster_assignment != cluster_assignment    )
									if verbose:
												print(
												    F"""    {num_changed:5d} elements changed their cluster assignment."""    )

        # Record heterogeneity convergence metric
						if record_heterogeneity is not None:
									# YOUR CODE HERE
									lowerCAmelCase_      =     compute_heterogeneity(__lowerCAmelCase   ,							__lowerCAmelCase   ,							__lowerCAmelCase   ,							__lowerCAmelCase    )
									record_heterogeneity.append(__lowerCAmelCase    )

						lowerCAmelCase_      =     cluster_assignment[:]

			return centroids, cluster_assignment


# Mock test below
if False:  # change to true to run this test case.
			from sklearn import datasets as ds

			_A       =					ds.load_iris()
			_A       =					3
			_A       =					[]
			_A       =					get_initial_centroids(dataset["data"], k, seed=0)
			_A,				_A       =					kmeans(
			    dataset["data"],
			    k,
			    initial_centroids,
			    maxiter=4_00,
			    record_heterogeneity=heterogeneity,
			    verbose=True,
			)
			plot_heterogeneity(heterogeneity, k)






def        lowerCamelCase__    (    __lowerCAmelCase  :  pd.DataFrame   ,							__lowerCAmelCase  :  np.ndarray   ,							__lowerCAmelCase  :  Union[str, Any]=None    ):




			"""simple docstring"""


			if fill_missing_report:
						lowerCAmelCase_      =     df.fillna(value=__lowerCAmelCase    )
			lowerCAmelCase_      =     1
			lowerCAmelCase_      =     df.select_dtypes(np.number    ).columns
			lowerCAmelCase_      =     (
			    df.groupby(["Cluster"]    )[  # construct report dataframe
			        numeric_cols
			    ]  # group by cluster number
			    .agg(
			        [
			            ("sum", np.sum),
			            ("mean_with_zeros", lambda __lowerCAmelCase    : np.mean(np.nan_to_num(__lowerCAmelCase    )    )),
			            ("mean_without_zeros", lambda __lowerCAmelCase    : x.replace(0   ,							np.NaN    ).mean()),
			            (
			                "mean_25-75",
			                lambda __lowerCAmelCase    : np.mean(
			                    np.nan_to_num(
			                        sorted(__lowerCAmelCase    )[
			                            round(len(__lowerCAmelCase    ) * 25 / 100    ) : round(len(__lowerCAmelCase    ) * 75 / 100    )
			                        ]    )    ),
			            ),
			            ("mean_with_na", np.mean),
			            ("min", lambda __lowerCAmelCase    : x.min()),
			            ("5%", lambda __lowerCAmelCase    : x.quantile(0.05    )),
			            ("25%", lambda __lowerCAmelCase    : x.quantile(0.25    )),
			            ("50%", lambda __lowerCAmelCase    : x.quantile(0.50    )),
			            ("75%", lambda __lowerCAmelCase    : x.quantile(0.75    )),
			            ("95%", lambda __lowerCAmelCase    : x.quantile(0.95    )),
			            ("max", lambda __lowerCAmelCase    : x.max()),
			            ("count", lambda __lowerCAmelCase    : x.count()),
			            ("stdev", lambda __lowerCAmelCase    : x.std()),
			            ("mode", lambda __lowerCAmelCase    : x.mode()[0]),
			            ("median", lambda __lowerCAmelCase    : x.median()),
			            ("# > 0", lambda __lowerCAmelCase    : (x > 0).sum()),
			        ]    )
			    .T.reset_index()
			    .rename(index=__lowerCAmelCase   ,							columns={"level_0": "Features", "level_1": "Type"}    )
			)  # rename columns
			# calculate the size of cluster(count of clientID's)
			lowerCAmelCase_      =     report[
			    (report["Features"] == "dummy") & (report["Type"] == "count")
			].copy()  # avoid SettingWithCopyWarning
			lowerCAmelCase_      =     (
			    "ClusterSize"  # rename created cluster df to match report column names
			)
			lowerCAmelCase_      =     "# of Customers"
			lowerCAmelCase_      =     pd.DataFrame(
			    clustersize.iloc[:, 2:].values
			    / clustersize.iloc[:, 2:].values.sum()    )
			lowerCAmelCase_      =     "% of Customers"  # rename created cluster df to match report column names
			lowerCAmelCase_      =     "ClusterProportion"
			lowerCAmelCase_      =     clusterproportion.columns.tolist()
			lowerCAmelCase_      =     cols[-2:] + cols[:-2]
			lowerCAmelCase_      =     clusterproportion[cols]  # rearrange columns to match report
			lowerCAmelCase_      =     report.columns
			lowerCAmelCase_      =     pd.DataFrame(
			    abs(
			        report[report["Type"] == "count"].iloc[:, 2:].values
			        - clustersize.iloc[:, 2:].values    )    )  # generating df with count of nan values
			lowerCAmelCase_      =     0
			lowerCAmelCase_      =     "# of nan"
			lowerCAmelCase_      =     report[
			    report["Type"] == "count"
			].Features.tolist()  # filling values in order to match report
			lowerCAmelCase_      =     a.columns.tolist()
			lowerCAmelCase_      =     cols[-2:] + cols[:-2]
			lowerCAmelCase_      =     a[cols]  # rearrange columns to match report
			lowerCAmelCase_      =     report.columns  # rename columns to match report
			lowerCAmelCase_      =     report.drop(
			    report[report.Type == "count"].index    )  # drop count values except cluster size
			lowerCAmelCase_      =     pd.concat(
			    [report, a, clustersize, clusterproportion]   ,							axis=0    )  # concat report with clustert size and nan values
			lowerCAmelCase_      =     report["Features"].isin(__lowerCAmelCase    )
			lowerCAmelCase_      =     report.columns.tolist()
			lowerCAmelCase_      =     cols[0:2] + cols[-1:] + cols[2:-1]
			lowerCAmelCase_      =     report[cols]
			lowerCAmelCase_      =     {
			    "ClusterSize": 9,
			    "ClusterProportion": 8,
			    "mean_with_zeros": 7,
			    "mean_with_na": 6,
			    "max": 5,
			    "50%": 4,
			    "min": 3,
			    "25%": 2,
			    "75%": 1,
			    "# of nan": 0,
			    "# > 0": -1,
			    "sum_with_na": -2,
			}
			lowerCAmelCase_      =     (
			    report.assign(
			        Sortera=lambda __lowerCAmelCase    : x.Type.map(__lowerCAmelCase    )   ,							Sortera=lambda __lowerCAmelCase    : list(reversed(range(len(__lowerCAmelCase    )    )    )    )   ,							)
			    .sort_values(["Sorter1", "Mark", "Sorter2"]   ,							ascending=__lowerCAmelCase    )
			    .drop(["Sorter1", "Sorter2"]   ,							axis=1    )
			)
			lowerCAmelCase_      =     ""
			lowerCAmelCase_      =     report.reset_index()
			lowerCAmelCase_      =     report.drop(columns=["index"]    )
			return report


if __name__ == "__main__":
			import doctest

			doctest.testmod()





import json
import os
from dataclasses import dataclass

import numpy as np
import pyarrow as pa

import datasets
from utils import get_duration


_A       =					10_00_00_00_00_00
_A       =					1_00_00

_A,				_A       =					os.path.split(__file__)
_A       =					os.path.join(RESULTS_BASEPATH, "results", RESULTS_FILENAME.replace(".py", ".json"))






def        lowerCamelCase__    (    __lowerCAmelCase  :  int   ,							__lowerCAmelCase  :  int    ):




			"""simple docstring"""


			lowerCAmelCase_      =     pa.Table.from_pydict({"col": [0] * chunk_size}    )
			lowerCAmelCase_      =     pa.concat_tables([table] * (num_examples // chunk_size)    )
			return datasets.Dataset(__lowerCAmelCase   ,							fingerprint="table_100B"    )





@dataclass
class      _lowerCAmelCase :
				_lowercase      		=42
				_lowercase      		=42
				_lowercase      		=42
				_lowercase      		=42




				def 					__a						( self					)    ->				List[Any]:
							lowerCAmelCase_      =     np.random.default_rng(self.seed					)
							lowerCAmelCase_      =     rng.integers(low=self.low    ,    high=self.high    ,    size=self.size					).tolist()




				def __iter__( self					)    ->				List[str]:
							return iter(self._sampled_values					)




				def __len__( self					)    ->				Optional[int]:
							return self.size






@get_duration
def        lowerCamelCase__    (    __lowerCAmelCase  :  datasets.Dataset    ):




			"""simple docstring"""


			lowerCAmelCase_      =     dataset[0]






@get_duration
def        lowerCamelCase__    (    __lowerCAmelCase  :  datasets.Dataset    ):




			"""simple docstring"""


			lowerCAmelCase_      =     dataset[-1]






@get_duration
def        lowerCamelCase__    (    __lowerCAmelCase  :  datasets.Dataset    ):




			"""simple docstring"""


			lowerCAmelCase_      =     dataset[range(len(__lowerCAmelCase    ) // 2   ,							len(__lowerCAmelCase    ) // 2 + 1024    )]






@get_duration
def        lowerCamelCase__    (    __lowerCAmelCase  :  datasets.Dataset    ):




			"""simple docstring"""


			lowerCAmelCase_      =     dataset[RandIter(0   ,							len(__lowerCAmelCase    )   ,							1024   ,							seed=42    )]






def        lowerCamelCase__    (    ):




			"""simple docstring"""


			lowerCAmelCase_      =     {"num examples": SPEED_TEST_N_EXAMPLES}
			lowerCAmelCase_      =     (get_first_row, get_last_row, get_batch_of_aaaa_rows, get_batch_of_aaaa_random_rows)
			print("generating dataset"    )
			lowerCAmelCase_      =     generate_100B_dataset(num_examples=__lowerCAmelCase   ,							chunk_size=__lowerCAmelCase    )
			print("Functions"    )
			for func in functions:
						print(func.__name__    )
						lowerCAmelCase_      =     func(__lowerCAmelCase    )

			with open(__lowerCAmelCase   ,							"wb"    ) as f:
						f.write(json.dumps(__lowerCAmelCase    ).encode("utf-8"    )    )


if __name__ == "__main__":  # useful to run the profiler
			benchmark_table_100B()



