






import jax
import jax.numpy as jnp
from bigbird_flax import FlaxBigBirdForNaturalQuestions
from datasets import load_from_disk

from transformers import BigBirdTokenizerFast


__a :    Dict								=				{0: "null", 1: "short", 2: "long", 3: "yes", 4: "no"}
__a :    Dict								=				set("".join(["‘", "’", "´", "`", ".", ",", "-", "\""]))



def      _SCREAMING_SNAKE_CASE  (	__lowercase				:     Any     ,			__lowercase				:     Any=0     ,			__lowercase				:     Dict=None      )  -> Dict:





    """simple docstring"""

    return [" ".join(x.split(""" """      )[begin:end]      ) for x in answers if len(x.split(""" """      )      ) > 1]



def      _SCREAMING_SNAKE_CASE  (	__lowercase				:     Optional[Any]     ,			__lowercase				:     str=False      )  -> List[str]:





    """simple docstring"""

    if make_sub_answers:
        # if answers are longer than one word, make sure a predictions is correct if it coresponds to the complete 1: or :-1 sub word
        # *e.g.* if the correct answer contains a prefix such as "the", or "a"
        __A             =		(
            given_answers + get_sub_answers(__lowercase     ,			begin=1      ) + get_sub_answers(__lowercase     ,			end=-1      )
        )
    __A             =		[]
    for answer in given_answers:
        __A             =		answer.replace("""_"""     ,			""" """      ).lower()
        __A             =		"".join(c if c not in PUNCTUATION_SET_TO_EXCLUDE else """ """ for c in alias      )
        answers.append(""" """.join(alias.split()      ).strip()      )
    return set(__lowercase      )



def      _SCREAMING_SNAKE_CASE  (	__lowercase				:     Optional[int]     ,			__lowercase				:     Tuple     ,			__lowercase				:     Optional[Any]=1     ,			__lowercase				:     Union[str, Any]=1_0_0      )  -> Optional[int]:





    """simple docstring"""

    __A             =		jax.lax.top_k(__lowercase     ,			__lowercase      )
    __A             =		jax.lax.top_k(__lowercase     ,			__lowercase      )

    __A             =		best_end_idx[:, None] - best_start_idx[None, :]
    __A             =		jnp.logical_or(widths < 0     ,			widths > max_size      )
    __A             =		(best_end_scores[:, None] + best_start_scores[None, :]) - (1E8 * mask)
    __A             =		jnp.argmax(__lowercase      ).item()

    return best_start_idx[best_score % top_k], best_end_idx[best_score // top_k]



def      _SCREAMING_SNAKE_CASE  (	__lowercase				:     str      )  -> List[str]:





    """simple docstring"""

    __A             =		sample["question"]["text"]
    __A             =		sample["document"]["tokens"]["token"]
    __A             =		sample["document"]["tokens"]["is_html"]
    __A             =		sample["annotations"]["long_answer"]
    __A             =		sample["annotations"]["short_answers"]

    __A             =		" ".join([context[i] for i in range(len(__lowercase      )      ) if not is_html[i]]      )

    # 0 - No ; 1 - Yes
    for answer in sample["annotations"]["yes_no_answer"]:
        if answer == 0 or answer == 1:
            return {
                "question": question,
                "context": context_string,
                "short": [],
                "long": [],
                "category": "no" if answer == 0 else "yes",
            }

    __A             =		[]
    for s in short_answers:
        short_targets.extend(s["""text"""]      )
    __A             =		list(set(__lowercase      )      )

    __A             =		[]
    for s in long_answers:
        if s["start_token"] == -1:
            continue
        __A             =		context[s["start_token"] : s["end_token"]]
        __A             =		is_html[s["start_token"] : s["end_token"]]
        __A             =		" ".join([answer[i] for i in range(len(__lowercase      )      ) if not html[i]]      )
        if new_answer not in long_targets:
            long_targets.append(__lowercase      )

    __A             =		"long_short" if len(short_targets + long_targets      ) > 0 else "null"

    return {
        "question": question,
        "context": context_string,
        "short": short_targets,
        "long": long_targets,
        "category": category,
    }



def      _SCREAMING_SNAKE_CASE  (	)  -> Optional[Any]:





    """simple docstring"""

    __A             =		load_from_disk("""natural-questions-validation"""      )
    __A             =		dataset.map(__lowercase      ).remove_columns(["""annotations""", """document""", """id"""]      )
    print(__lowercase      )

    __A             =		dataset.filter(lambda __lowercase      : (len(x["""question"""]      ) + len(x["""context"""]      )) < 4 * 4_0_9_6      )
    __A             =		short_validation_dataset.filter(lambda __lowercase      : x["category"] != "null"      )
    short_validation_dataset

    __A             =		"vasudevgupta/flax-bigbird-natural-questions"
    __A             =		FlaxBigBirdForNaturalQuestions.from_pretrained(__lowercase      )
    __A             =		BigBirdTokenizerFast.from_pretrained(__lowercase      )

    @jax.jit
    def forward(*__lowercase				:     Dict     ,			**__lowercase				:     List[str]      ):
        __A             =		model(*__lowercase     ,			**__lowercase      )
        return start_logits, end_logits, jnp.argmax(__lowercase     ,			axis=-1      )

    def evaluate(__lowercase				:     str      ):
        # encode question and context so that they are separated by a tokenizer.sep_token and cut at max_length
        __A             =		tokenizer(
            example["""question"""]     ,			example["""context"""]     ,			return_tensors="""np"""     ,			max_length=4_0_9_6     ,			padding="""max_length"""     ,			truncation=__lowercase     ,			)

        __A             =		forward(**__lowercase      )

        __A             =		CATEGORY_MAPPING[category.item()]

        __A             =		example["long"] + example["short"]
        if example["category"] in ["yes", "no", "null"]:
            __A             =		[example["category"]]
        __A             =		example["category"] != "null"
        # Now target can be: "yes", "no", "null", "list of long & short answers"

        if predicted_category in ["yes", "no", "null"]:
            __A             =		[predicted_category]
            __A             =		example["output"] == example["targets"]
            __A             =		predicted_category != "null"
            return example

        __A             =		3_8 if predicted_category == "short" else 1_0_2_4
        __A             =		get_best_valid_start_end_idx(
            start_scores[0]     ,			end_scores[0]     ,			top_k=8     ,			max_size=__lowercase      )

        __A             =		inputs["input_ids"][0].tolist()
        __A             =		[tokenizer.decode(input_ids[start_score : end_score + 1]      )]

        __A             =		expand_to_aliases(example["""targets"""]     ,			make_sub_answers=__lowercase      )
        __A             =		expand_to_aliases(example["""output"""]      )

        # some preprocessing to both prediction and answer
        __A             =		{"".join(a.split()      ) for a in answers}
        __A             =		{"".join(p.split()      ) for p in predictions}
        __A             =		{s for s in predictions if s not in ["``", "''", "`", "'"]}

        # if there is a common element, it's a exact match
        __A             =		len(list(answers & predictions      )      ) > 0
        __A             =		predicted_category != "null" and len(__lowercase      ) > 0

        return example

    __A             =		short_validation_dataset.map(__lowercase      )

    __A             =		len(__lowercase      )
    __A             =		len(short_validation_dataset.filter(lambda __lowercase      : x["match"] == 1      )      )
    print("""EM score:"""     ,			(matched / total) * 1_0_0     ,			"""%"""      )


if __name__ == "__main__":
      main()












import importlib
import inspect
from functools import wraps
from typing import TYPE_CHECKING, Optional, Union

from .download.streaming_download_manager import (
    xbasename,
    xdirname,
    xet_parse,
    xexists,
    xgetsize,
    xglob,
    xgzip_open,
    xisdir,
    xisfile,
    xjoin,
    xlistdir,
    xnumpy_load,
    xopen,
    xpandas_read_csv,
    xpandas_read_excel,
    xPath,
    xrelpath,
    xsio_loadmat,
    xsplit,
    xsplitext,
    xwalk,
    xxml_dom_minidom_parse,
)
from .utils.logging import get_logger
from .utils.patching import patch_submodule
from .utils.py_utils import get_imports


__a :    List[str]								=				get_logger(__name__)


if TYPE_CHECKING:
  from .builder import DatasetBuilder



def      _SCREAMING_SNAKE_CASE  (	__lowercase				:     Dict     ,			__lowercase				:     Optional[Union[str, bool]] = None      )  -> List[str]:





    """simple docstring"""

    __A             =		importlib.import_module(__lowercase      )

    # TODO(QL): always update the module to add subsequent new authentication
    if hasattr(__lowercase     ,			"""_patched_for_streaming"""      ) and module._patched_for_streaming:
        return

    def wrap_auth(__lowercase				:     int      ):
        @wraps(__lowercase      )
        def wrapper(*__lowercase				:     Tuple     ,			**__lowercase				:     List[str]      ):
            return function(*__lowercase     ,			use_auth_token=__lowercase     ,			**__lowercase      )

        __A             =		"""wrap_auth"""
        return wrapper

    # open files in a streaming fashion
    patch_submodule(__lowercase     ,			"""open"""     ,			wrap_auth(__lowercase      )      ).start()
    patch_submodule(__lowercase     ,			"""os.listdir"""     ,			wrap_auth(__lowercase      )      ).start()
    patch_submodule(__lowercase     ,			"""os.walk"""     ,			wrap_auth(__lowercase      )      ).start()
    patch_submodule(__lowercase     ,			"""glob.glob"""     ,			wrap_auth(__lowercase      )      ).start()
    # allow to navigate in remote zip files
    patch_submodule(__lowercase     ,			"""os.path.join"""     ,			__lowercase      ).start()
    patch_submodule(__lowercase     ,			"""os.path.dirname"""     ,			__lowercase      ).start()
    patch_submodule(__lowercase     ,			"""os.path.basename"""     ,			__lowercase      ).start()
    patch_submodule(__lowercase     ,			"""os.path.relpath"""     ,			__lowercase      ).start()
    patch_submodule(__lowercase     ,			"""os.path.split"""     ,			__lowercase      ).start()
    patch_submodule(__lowercase     ,			"""os.path.splitext"""     ,			__lowercase      ).start()
    # allow checks on paths
    patch_submodule(__lowercase     ,			"""os.path.exists"""     ,			wrap_auth(__lowercase      )      ).start()
    patch_submodule(__lowercase     ,			"""os.path.isdir"""     ,			wrap_auth(__lowercase      )      ).start()
    patch_submodule(__lowercase     ,			"""os.path.isfile"""     ,			wrap_auth(__lowercase      )      ).start()
    patch_submodule(__lowercase     ,			"""os.path.getsize"""     ,			wrap_auth(__lowercase      )      ).start()
    patch_submodule(__lowercase     ,			"""pathlib.Path"""     ,			__lowercase      ).start()
    # file readers
    patch_submodule(__lowercase     ,			"""gzip.open"""     ,			wrap_auth(__lowercase      )      ).start()
    patch_submodule(__lowercase     ,			"""numpy.load"""     ,			wrap_auth(__lowercase      )      ).start()
    patch_submodule(__lowercase     ,			"""pandas.read_csv"""     ,			wrap_auth(__lowercase      )     ,			attrs=["""__version__"""]      ).start()
    patch_submodule(__lowercase     ,			"""pandas.read_excel"""     ,			wrap_auth(__lowercase      )     ,			attrs=["""__version__"""]      ).start()
    patch_submodule(__lowercase     ,			"""scipy.io.loadmat"""     ,			wrap_auth(__lowercase      )     ,			attrs=["""__version__"""]      ).start()
    patch_submodule(__lowercase     ,			"""xml.etree.ElementTree.parse"""     ,			wrap_auth(__lowercase      )      ).start()
    patch_submodule(__lowercase     ,			"""xml.dom.minidom.parse"""     ,			wrap_auth(__lowercase      )      ).start()
    __A             =		True






def      _SCREAMING_SNAKE_CASE  (	__lowercase				:     "DatasetBuilder"      )  -> Union[str, Any]:





    """simple docstring"""

    extend_module_for_streaming(builder.__module__     ,			use_auth_token=builder.use_auth_token      )
    # if needed, we also have to extend additional internal imports (like wmt14 -> wmt_utils)
    if not builder.__module__.startswith("""datasets."""      ):  # check that it's not a packaged builder like csv
        for imports in get_imports(inspect.getfile(builder.__class__      )      ):
            if imports[0] == "internal":
                __A             =		imports[1]
                __A             =		""".""".join(builder.__module__.split("""."""      )[:-1] + [internal_import_name]      )
                extend_module_for_streaming(__lowercase     ,			use_auth_token=builder.use_auth_token      )

    # builders can inherit from other builders that might use streaming functionality
    # (for example, ImageFolder and AudioFolder inherit from FolderBuilder which implements examples generation)
    # but these parents builders are not patched automatically as they are not instantiated, so we patch them here
    from .builder import DatasetBuilder

    __A             =		[
        cls.__module__
        for cls in type(__lowercase      ).__mro__[1:]  # make sure it's not the same module we've already patched
        if issubclass(cls     ,			__lowercase      ) and cls.__module__ != DatasetBuilder.__module__
    ]  # check it's not a standard builder from datasets.builder
    for module in parent_builder_modules:
        extend_module_for_streaming(__lowercase     ,			use_auth_token=builder.use_auth_token      )




