




'''simple docstring'''







# Copyright 2022 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from typing import List





def 						snake_case_							(_a     :							List[str]   ,       _a     :							str					):
    if name != "training_function" and name != "main":
        raise ValueError(F"Incorrect function name passed: {name}, choose either 'main' or 'training_function'"					)
    UpperCAmelCase	,					UpperCAmelCase      	=		[], False
    for line in lines:
        if not found_start and F"def {name}" in line:
            UpperCAmelCase      	=		True
            good_lines.append(_a					)
            continue
        if found_start:
            if name == "training_function" and "def main" in line:
                return good_lines
            if name == "main" and "if __name__" in line:
                return good_lines
            good_lines.append(_a					)





def 						snake_case_							(_a     :							List[str]					):
    return [line for line in lines if not line.lstrip().startswith('''#'''					) and line != "\n"]







def 						snake_case_							(_a     :							str   ,       _a     :							str   ,       _a     :							bool   ,       _a     :							str = None					):
    with open(_a   ,       '''r'''					) as f:
        UpperCAmelCase      	=		f.readlines()
    with open(os.path.abspath(os.path.join('''examples'''   ,       '''nlp_example.py'''					)					)   ,       '''r'''					) as f:
        UpperCAmelCase      	=		f.readlines()
    with open(_a   ,       '''r'''					) as f:
        UpperCAmelCase      	=		f.readlines()
    if secondary_filename is not None:
        with open(_a   ,       '''r'''					) as f:
            UpperCAmelCase      	=		f.readlines()

    # This is our base, we remove all the code from here in our `full_filename` and `feature_filename` to find the new content
    if parser_only:
        UpperCAmelCase      	=		clean_lines(get_function_contents_by_name(_a   ,       '''main'''					)					)
        UpperCAmelCase      	=		clean_lines(get_function_contents_by_name(_a   ,       '''main'''					)					)
        UpperCAmelCase      	=		clean_lines(get_function_contents_by_name(_a   ,       '''main'''					)					)
        if secondary_filename is not None:
            UpperCAmelCase      	=		clean_lines(get_function_contents_by_name(_a   ,       '''main'''					)					)
    else:
        UpperCAmelCase      	=		clean_lines(get_function_contents_by_name(_a   ,       '''training_function'''					)					)
        UpperCAmelCase      	=		clean_lines(get_function_contents_by_name(_a   ,       '''training_function'''					)					)
        UpperCAmelCase      	=		clean_lines(get_function_contents_by_name(_a   ,       '''training_function'''					)					)
        if secondary_filename is not None:
            UpperCAmelCase      	=		clean_lines(
                get_function_contents_by_name(_a   ,       '''training_function'''					)					)

    UpperCAmelCase      	=		'''train_dataloader, eval_dataloader = get_dataloaders(accelerator, batch_size)\n'''

    # Specific code in our script that differs from the full version, aka what is new
    UpperCAmelCase      	=		[]
    UpperCAmelCase      	=		[]  # We keep track of the idxs just in case it's a repeated statement
    UpperCAmelCase      	=		iter(_a					)
    for i in range(len(_a					) - 1					):
        if i not in passed_idxs:
            UpperCAmelCase      	=		next(_a					)
            if (line not in full_file_func) and (line.lstrip() != _dl_line):
                if "TESTING_MOCKED_DATALOADERS" not in line:
                    new_feature_code.append(_a					)
                    passed_idxs.append(_a					)
                else:
                    # Skip over the `config['num_epochs'] = 2` statement
                    UpperCAmelCase      	=		next(_a					)

    # Extract out just the new parts from the full_file_training_func
    UpperCAmelCase      	=		[]
    UpperCAmelCase      	=		[]  # We keep track of the idxs just in case it's a repeated statement
    for i, line in enumerate(_a					):
        if i not in passed_idxs:
            if (line not in full_file_func) and (line.lstrip() != _dl_line):
                if "TESTING_MOCKED_DATALOADERS" not in line:
                    new_full_example_parts.append(_a					)
                    passed_idxs.append(_a					)

    # Finally, get the overall diff
    UpperCAmelCase      	=		[line for line in new_feature_code if line not in new_full_example_parts]
    if secondary_filename is not None:
        UpperCAmelCase      	=		[line for line in full_file_contents if line not in secondary_file_func]
        UpperCAmelCase      	=		[line for line in diff_from_example if line not in diff_from_two]

    return diff_from_example










'''simple docstring'''







import multiprocessing
import os
from typing import BinaryIO, Optional, Union

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.csv.csv import Csv
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader


class       _a  (     __a			):
       def __init__(     self						:     Any       ,    lowercase						:     NestedDataStructureLike[PathLike]       ,    lowercase						:     Optional[NamedSplit] = None       ,    lowercase						:     Optional[Features] = None       ,    lowercase						:     str = None       ,    lowercase						:     bool = False       ,    lowercase						:     bool = False       ,    lowercase						:     Optional[int] = None       ,    **lowercase						:     str       ,    ):



           '''simple docstring'''

           super().__init__(
               lowercase       ,    split=lowercase       ,    features=lowercase       ,    cache_dir=lowercase       ,    keep_in_memory=lowercase       ,    streaming=lowercase       ,    num_proc=lowercase       ,    **lowercase       ,    )
           UpperCAmelCase      	=		path_or_paths if isinstance(lowercase       ,    lowercase    ) else {self.split: path_or_paths}
           UpperCAmelCase      	=		Csv(
               cache_dir=lowercase       ,    data_files=lowercase       ,    features=lowercase       ,    **lowercase       ,    )







       def    A				(     self						:     Tuple    ):



           '''simple docstring'''

           if self.streaming:
               UpperCAmelCase      	=		self.builder.as_streaming_dataset(split=self.split    )
           # Build regular (map-style) dataset
           else:
               UpperCAmelCase      	=		None
               UpperCAmelCase      	=		None
               UpperCAmelCase      	=		None
               UpperCAmelCase      	=		None

               self.builder.download_and_prepare(
                   download_config=lowercase       ,    download_mode=lowercase       ,    verification_mode=lowercase       ,    base_path=lowercase       ,    num_proc=self.num_proc       ,    )
               UpperCAmelCase      	=		self.builder.as_dataset(
                   split=self.split       ,    verification_mode=lowercase       ,    in_memory=self.keep_in_memory    )
           return dataset






class       _a  :
       def __init__(     self						:     Optional[int]       ,    lowercase						:     Dataset       ,    lowercase						:     Union[PathLike, BinaryIO]       ,    lowercase						:     Optional[int] = None       ,    lowercase						:     Optional[int] = None       ,    **lowercase						:     int       ,    ):



           '''simple docstring'''

           if num_proc is not None and num_proc <= 0:
               raise ValueError(f"num_proc {num_proc} must be an integer > 0."    )

           UpperCAmelCase      	=		dataset
           UpperCAmelCase      	=		path_or_buf
           UpperCAmelCase      	=		batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
           UpperCAmelCase      	=		num_proc
           UpperCAmelCase      	=		'''utf-8'''
           UpperCAmelCase      	=		to_csv_kwargs
       def    A				(     self						:     Union[str, Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.to_csv_kwargs.pop('''path_or_buf'''       ,    lowercase    )
           UpperCAmelCase      	=		self.to_csv_kwargs.pop('''header'''       ,    lowercase    )
           UpperCAmelCase      	=		self.to_csv_kwargs.pop('''index'''       ,    lowercase    )

           if isinstance(self.path_or_buf       ,    (str, bytes, os.PathLike)    ):
               with open(self.path_or_buf       ,    '''wb+'''    ) as buffer:
                   UpperCAmelCase      	=		self._write(file_obj=lowercase       ,    header=lowercase       ,    index=lowercase       ,    **self.to_csv_kwargs    )
           else:
               UpperCAmelCase      	=		self._write(file_obj=self.path_or_buf       ,    header=lowercase       ,    index=lowercase       ,    **self.to_csv_kwargs    )
           return written
       def    A				(     self						:     List[str]       ,    lowercase						:     Dict    ):



           '''simple docstring'''

           UpperCAmelCase	,					UpperCAmelCase	,					UpperCAmelCase	,					UpperCAmelCase      	=		args

           UpperCAmelCase      	=		query_table(
               table=self.dataset.data       ,    key=slice(lowercase       ,    offset + self.batch_size    )       ,    indices=self.dataset._indices       ,    )
           UpperCAmelCase      	=		batch.to_pandas().to_csv(
               path_or_buf=lowercase       ,    header=header if (offset == 0) else False       ,    index=lowercase       ,    **lowercase    )
           return csv_str.encode(self.encoding    )







       def    A				(     self						:     List[str]       ,    lowercase						:     BinaryIO       ,    lowercase						:     int       ,    lowercase						:     List[str]       ,    **lowercase						:     List[Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		0

           if self.num_proc is None or self.num_proc == 1:
               for offset in logging.tqdm(
                   range(0       ,    len(self.dataset    )       ,    self.batch_size    )       ,    unit='''ba'''       ,    disable=not logging.is_progress_bar_enabled()       ,    desc='''Creating CSV from Arrow format'''       ,    ):
                   UpperCAmelCase      	=		self._batch_csv((offset, header, index, to_csv_kwargs)    )
                   written += file_obj.write(lowercase    )

           else:
               UpperCAmelCase	,					UpperCAmelCase      	=		len(self.dataset    ), self.batch_size
               with multiprocessing.Pool(self.num_proc    ) as pool:
                   for csv_str in logging.tqdm(
                       pool.imap(
                           self._batch_csv       ,    [(offset, header, index, to_csv_kwargs) for offset in range(0       ,    lowercase       ,    lowercase    )]       ,    )       ,    total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size       ,    unit='''ba'''       ,    disable=not logging.is_progress_bar_enabled()       ,    desc='''Creating CSV from Arrow format'''       ,    ):
                       written += file_obj.write(lowercase    )

           return written




