






import json
import os
from dataclasses import dataclass

import numpy as np
import pyarrow as pa

import datasets
from utils import get_duration


__a :    int								=				100000000000
__a :    int								=				10000

__a					,__a :    str								=				os.path.split(__file__)
__a :    str								=				os.path.join(RESULTS_BASEPATH, "results", RESULTS_FILENAME.replace(".py", ".json"))



def      _SCREAMING_SNAKE_CASE  (	__lowercase				:     int     ,			__lowercase				:     int      )  -> datasets.Dataset:





    """simple docstring"""

    __A             =		pa.Table.from_pydict({"""col""": [0] * chunk_size}      )
    __A             =		pa.concat_tables([table] * (num_examples // chunk_size)      )
    return datasets.Dataset(__lowercase     ,			fingerprint="""table_100B"""      )


@dataclass
class  __lowercase :







     '''simple docstring'''
     SCREAMING_SNAKE_CASE				  =			42
     SCREAMING_SNAKE_CASE				  =			42
     SCREAMING_SNAKE_CASE				  =			42
     SCREAMING_SNAKE_CASE				  =			42




     def 						lowerCAmelCase_       (			self    :     Dict			):


         """simple docstring"""



         __A             =		np.random.default_rng(self.seed			)
         __A             =		rng.integers(low=self.low	,		high=self.high	,		size=self.size			).tolist()




     def __iter__(			self    :     Optional[Any]			):


         """simple docstring"""



         return iter(self._sampled_values			)





     def __len__(			self    :     Any			):


         """simple docstring"""



         return self.size



@get_duration
def      _SCREAMING_SNAKE_CASE  (	__lowercase				:     datasets.Dataset      )  -> List[Any]:





    """simple docstring"""

    __A             =		dataset[0]



@get_duration
def      _SCREAMING_SNAKE_CASE  (	__lowercase				:     datasets.Dataset      )  -> List[str]:





    """simple docstring"""

    __A             =		dataset[-1]



@get_duration
def      _SCREAMING_SNAKE_CASE  (	__lowercase				:     datasets.Dataset      )  -> List[str]:





    """simple docstring"""

    __A             =		dataset[range(len(__lowercase      ) // 2     ,			len(__lowercase      ) // 2 + 1_0_2_4      )]



@get_duration
def      _SCREAMING_SNAKE_CASE  (	__lowercase				:     datasets.Dataset      )  -> Tuple:





    """simple docstring"""

    __A             =		dataset[RandIter(0     ,			len(__lowercase      )     ,			1_0_2_4     ,			seed=4_2      )]



def      _SCREAMING_SNAKE_CASE  (	)  -> Optional[int]:





    """simple docstring"""

    __A             =		{"""num examples""": SPEED_TEST_N_EXAMPLES}
    __A             =		(get_first_row, get_last_row, get_batch_of_aaaa_rows, get_batch_of_aaaa_random_rows)
    print("""generating dataset"""      )
    __A             =		generate_100B_dataset(num_examples=__lowercase     ,			chunk_size=__lowercase      )
    print("""Functions"""      )
    for func in functions:
        print(func.__name__      )
        __A             =		func(__lowercase      )

    with open(__lowercase     ,			"""wb"""      ) as f:
        f.write(json.dumps(__lowercase      ).encode("""utf-8"""      )      )


if __name__ == "__main__":  # useful to run the profiler
  benchmark_table_100B()












# Copyright 2022 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from typing import List



def      _SCREAMING_SNAKE_CASE  (	__lowercase				:     List[str]     ,			__lowercase				:     str      )  -> Optional[int]:





    """simple docstring"""

    if name != "training_function" and name != "main":
        raise ValueError(f"Incorrect function name passed: {name}, choose either 'main' or 'training_function'"      )
    __A      ,		__A             =		[], False
    for line in lines:
        if not found_start and f"def {name}" in line:
            __A             =		True
            good_lines.append(__lowercase      )
            continue
        if found_start:
            if name == "training_function" and "def main" in line:
                return good_lines
            if name == "main" and "if __name__" in line:
                return good_lines
            good_lines.append(__lowercase      )



def      _SCREAMING_SNAKE_CASE  (	__lowercase				:     List[str]      )  -> Optional[int]:





    """simple docstring"""

    return [line for line in lines if not line.lstrip().startswith("""#"""      ) and line != "\n"]






def      _SCREAMING_SNAKE_CASE  (	__lowercase				:     str     ,			__lowercase				:     str     ,			__lowercase				:     bool     ,			__lowercase				:     str = None      )  -> List[Any]:





    """simple docstring"""

    with open(__lowercase     ,			"""r"""      ) as f:
        __A             =		f.readlines()
    with open(os.path.abspath(os.path.join("""examples"""     ,			"""nlp_example.py"""      )      )     ,			"""r"""      ) as f:
        __A             =		f.readlines()
    with open(__lowercase     ,			"""r"""      ) as f:
        __A             =		f.readlines()
    if secondary_filename is not None:
        with open(__lowercase     ,			"""r"""      ) as f:
            __A             =		f.readlines()

    # This is our base, we remove all the code from here in our `full_filename` and `feature_filename` to find the new content
    if parser_only:
        __A             =		clean_lines(get_function_contents_by_name(__lowercase     ,			"""main"""      )      )
        __A             =		clean_lines(get_function_contents_by_name(__lowercase     ,			"""main"""      )      )
        __A             =		clean_lines(get_function_contents_by_name(__lowercase     ,			"""main"""      )      )
        if secondary_filename is not None:
            __A             =		clean_lines(get_function_contents_by_name(__lowercase     ,			"""main"""      )      )
    else:
        __A             =		clean_lines(get_function_contents_by_name(__lowercase     ,			"""training_function"""      )      )
        __A             =		clean_lines(get_function_contents_by_name(__lowercase     ,			"""training_function"""      )      )
        __A             =		clean_lines(get_function_contents_by_name(__lowercase     ,			"""training_function"""      )      )
        if secondary_filename is not None:
            __A             =		clean_lines(
                get_function_contents_by_name(__lowercase     ,			"""training_function"""      )      )

    __A             =		"""train_dataloader, eval_dataloader = get_dataloaders(accelerator, batch_size)\n"""

    # Specific code in our script that differs from the full version, aka what is new
    __A             =		[]
    __A             =		[]  # We keep track of the idxs just in case it's a repeated statement
    __A             =		iter(__lowercase      )
    for i in range(len(__lowercase      ) - 1      ):
        if i not in passed_idxs:
            __A             =		next(__lowercase      )
            if (line not in full_file_func) and (line.lstrip() != _dl_line):
                if "TESTING_MOCKED_DATALOADERS" not in line:
                    new_feature_code.append(__lowercase      )
                    passed_idxs.append(__lowercase      )
                else:
                    # Skip over the `config['num_epochs'] = 2` statement
                    __A             =		next(__lowercase      )

    # Extract out just the new parts from the full_file_training_func
    __A             =		[]
    __A             =		[]  # We keep track of the idxs just in case it's a repeated statement
    for i, line in enumerate(__lowercase      ):
        if i not in passed_idxs:
            if (line not in full_file_func) and (line.lstrip() != _dl_line):
                if "TESTING_MOCKED_DATALOADERS" not in line:
                    new_full_example_parts.append(__lowercase      )
                    passed_idxs.append(__lowercase      )

    # Finally, get the overall diff
    __A             =		[line for line in new_feature_code if line not in new_full_example_parts]
    if secondary_filename is not None:
        __A             =		[line for line in full_file_contents if line not in secondary_file_func]
        __A             =		[line for line in diff_from_example if line not in diff_from_two]

    return diff_from_example




