

'''simple docstring'''






import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import ConvNextConfig, ConvNextForImageClassification, ConvNextImageProcessor
from transformers.utils import logging


logging.set_verbosity_info()
snake_case_					  =	logging.get_logger(__name__)


def  __lowerCamelCase      (				SCREAMING_SNAKE_CASE_					:     List[str]   )							->       Union[str, Any]:
      SCREAMING_SNAKE_CASE_				:      Tuple					=	ConvNextConfig()

      if "tiny" in checkpoint_url:
            SCREAMING_SNAKE_CASE_				:      str					=	[3, 3, 9, 3]
            SCREAMING_SNAKE_CASE_				:      Any					=	[9_6, 1_9_2, 3_8_4, 7_6_8]
      if "small" in checkpoint_url:
            SCREAMING_SNAKE_CASE_				:      Any					=	[3, 3, 2_7, 3]
            SCREAMING_SNAKE_CASE_				:      Optional[int]					=	[9_6, 1_9_2, 3_8_4, 7_6_8]
      if "base" in checkpoint_url:
            SCREAMING_SNAKE_CASE_				:      Any					=	[3, 3, 2_7, 3]
            SCREAMING_SNAKE_CASE_				:      List[Any]					=	[1_2_8, 2_5_6, 5_1_2, 1_0_2_4]
      if "large" in checkpoint_url:
            SCREAMING_SNAKE_CASE_				:      str					=	[3, 3, 2_7, 3]
            SCREAMING_SNAKE_CASE_				:      Dict					=	[1_9_2, 3_8_4, 7_6_8, 1_5_3_6]
      if "xlarge" in checkpoint_url:
            SCREAMING_SNAKE_CASE_				:      List[str]					=	[3, 3, 2_7, 3]
            SCREAMING_SNAKE_CASE_				:      List[Any]					=	[2_5_6, 5_1_2, 1_0_2_4, 2_0_4_8]

      if "1k" in checkpoint_url:
            SCREAMING_SNAKE_CASE_				:      int					=	1_0_0_0
            SCREAMING_SNAKE_CASE_				:      int					=	"imagenet-1k-id2label.json"
            SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	(1, 1_0_0_0)
      else:
            SCREAMING_SNAKE_CASE_				:      Dict					=	2_1_8_4_1
            SCREAMING_SNAKE_CASE_				:      str					=	"imagenet-22k-id2label.json"
            SCREAMING_SNAKE_CASE_				:      str					=	(1, 2_1_8_4_1)

      SCREAMING_SNAKE_CASE_				:      Any					=	"huggingface/label-files"
      SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	num_labels
      SCREAMING_SNAKE_CASE_				:      List[str]					=	json.load(open(hf_hub_download(_lowerCAmelCase  ,  _lowerCAmelCase  ,  repo_type="dataset"   )  ,  "r"   )   )
      SCREAMING_SNAKE_CASE_				:      str					=	{int(_lowerCAmelCase   ): v for k, v in idalabel.items()}
      if "1k" not in checkpoint_url:
            # this dataset contains 21843 labels but the model only has 21841
            # we delete the classes as mentioned in https://github.com/google-research/big_transfer/issues/18
            del idalabel[9_2_0_5]
            del idalabel[1_5_0_2_7]
      SCREAMING_SNAKE_CASE_				:      Dict					=	idalabel
      SCREAMING_SNAKE_CASE_				:      Dict					=	{v: k for k, v in idalabel.items()}
      SCREAMING_SNAKE_CASE_				:      List[Any]					=	hidden_sizes
      SCREAMING_SNAKE_CASE_				:      List[Any]					=	depths

      return config, expected_shape


def  __lowerCamelCase      (				SCREAMING_SNAKE_CASE_					:     str   )							->       Union[str, Any]:
      if "downsample_layers.0.0" in name:
            SCREAMING_SNAKE_CASE_				:      Any					=	name.replace("downsample_layers.0.0"  ,  "embeddings.patch_embeddings"   )
      if "downsample_layers.0.1" in name:
            SCREAMING_SNAKE_CASE_				:      str					=	name.replace("downsample_layers.0.1"  ,  "embeddings.norm"   )  # we rename to layernorm later on
      if "downsample_layers.1.0" in name:
            SCREAMING_SNAKE_CASE_				:      Any					=	name.replace("downsample_layers.1.0"  ,  "stages.1.downsampling_layer.0"   )
      if "downsample_layers.1.1" in name:
            SCREAMING_SNAKE_CASE_				:      Tuple					=	name.replace("downsample_layers.1.1"  ,  "stages.1.downsampling_layer.1"   )
      if "downsample_layers.2.0" in name:
            SCREAMING_SNAKE_CASE_				:      List[Any]					=	name.replace("downsample_layers.2.0"  ,  "stages.2.downsampling_layer.0"   )
      if "downsample_layers.2.1" in name:
            SCREAMING_SNAKE_CASE_				:      Dict					=	name.replace("downsample_layers.2.1"  ,  "stages.2.downsampling_layer.1"   )
      if "downsample_layers.3.0" in name:
            SCREAMING_SNAKE_CASE_				:      Dict					=	name.replace("downsample_layers.3.0"  ,  "stages.3.downsampling_layer.0"   )
      if "downsample_layers.3.1" in name:
            SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	name.replace("downsample_layers.3.1"  ,  "stages.3.downsampling_layer.1"   )
      if "stages" in name and "downsampling_layer" not in name:
            # stages.0.0. for instance should be renamed to stages.0.layers.0.
            SCREAMING_SNAKE_CASE_				:      Any					=	name[: len("stages.0"   )] + ".layers" + name[len("stages.0"   ) :]
      if "stages" in name:
            SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	name.replace("stages"  ,  "encoder.stages"   )
      if "norm" in name:
            SCREAMING_SNAKE_CASE_				:      List[Any]					=	name.replace("norm"  ,  "layernorm"   )
      if "gamma" in name:
            SCREAMING_SNAKE_CASE_				:      Dict					=	name.replace("gamma"  ,  "layer_scale_parameter"   )
      if "head" in name:
            SCREAMING_SNAKE_CASE_				:      Any					=	name.replace("head"  ,  "classifier"   )

      return name


def  __lowerCamelCase      (				)							->       Dict:
      SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	"http://images.cocodataset.org/val2017/000000039769.jpg"
      SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	Image.open(requests.get(_lowerCAmelCase  ,  stream=_lowerCAmelCase   ).raw   )
      return im


@torch.no_grad()
def  __lowerCamelCase      (				SCREAMING_SNAKE_CASE_					:     int  ,  SCREAMING_SNAKE_CASE_					:     Dict   )							->       Any:
      SCREAMING_SNAKE_CASE_,						SCREAMING_SNAKE_CASE_				:      Any					=	get_convnext_config(_lowerCAmelCase   )
      # load original state_dict from URL
      SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	torch.hub.load_state_dict_from_url(_lowerCAmelCase   )["model"]
      # rename keys
      for key in state_dict.copy().keys():
            SCREAMING_SNAKE_CASE_				:      int					=	state_dict.pop(_lowerCAmelCase   )
            SCREAMING_SNAKE_CASE_				:      str					=	val
      # add prefix to all keys expect classifier head
      for key in state_dict.copy().keys():
            SCREAMING_SNAKE_CASE_				:      Tuple					=	state_dict.pop(_lowerCAmelCase   )
            if not key.startswith("classifier"   ):
                  SCREAMING_SNAKE_CASE_				:      Any					=	"convnext." + key
            SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	val

      # load HuggingFace model
      SCREAMING_SNAKE_CASE_				:      Optional[int]					=	ConvNextForImageClassification(_lowerCAmelCase   )
      model.load_state_dict(_lowerCAmelCase   )
      model.eval()

      # Check outputs on an image, prepared by ConvNextImageProcessor
      SCREAMING_SNAKE_CASE_				:      int					=	2_2_4 if "224" in checkpoint_url else 3_8_4
      SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	ConvNextImageProcessor(size=_lowerCAmelCase   )
      SCREAMING_SNAKE_CASE_				:      Optional[int]					=	image_processor(images=prepare_img()  ,  return_tensors="pt"   ).pixel_values

      SCREAMING_SNAKE_CASE_				:      Any					=	model(_lowerCAmelCase   ).logits

      # note: the logits below were obtained without center cropping
      if checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_tiny_1k_224_ema.pth":
            SCREAMING_SNAKE_CASE_				:      Optional[int]					=	torch.tensor([-0.1210, -0.6605, 0.1918]   )
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_small_1k_224_ema.pth":
            SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	torch.tensor([-0.4473, -0.1847, -0.6365]   )
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_base_1k_224_ema.pth":
            SCREAMING_SNAKE_CASE_				:      Tuple					=	torch.tensor([0.4525, 0.7539, 0.0308]   )
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_base_1k_384.pth":
            SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	torch.tensor([0.3561, 0.6350, -0.0384]   )
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_large_1k_224_ema.pth":
            SCREAMING_SNAKE_CASE_				:      List[str]					=	torch.tensor([0.4174, -0.0989, 0.1489]   )
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_large_1k_384.pth":
            SCREAMING_SNAKE_CASE_				:      List[str]					=	torch.tensor([0.2513, -0.1349, -0.1613]   )
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_base_22k_224.pth":
            SCREAMING_SNAKE_CASE_				:      List[Any]					=	torch.tensor([1.2980, 0.3631, -0.1198]   )
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_large_22k_224.pth":
            SCREAMING_SNAKE_CASE_				:      Dict					=	torch.tensor([1.2963, 0.1227, 0.1723]   )
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_xlarge_22k_224.pth":
            SCREAMING_SNAKE_CASE_				:      int					=	torch.tensor([1.7956, 0.8390, 0.2820]   )
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_base_22k_1k_224.pth":
            SCREAMING_SNAKE_CASE_				:      Optional[int]					=	torch.tensor([-0.2822, -0.0502, -0.0878]   )
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_base_22k_1k_384.pth":
            SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	torch.tensor([-0.5672, -0.0730, -0.4348]   )
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_large_22k_1k_224.pth":
            SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	torch.tensor([0.2681, 0.2365, 0.6246]   )
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_large_22k_1k_384.pth":
            SCREAMING_SNAKE_CASE_				:      List[Any]					=	torch.tensor([-0.2642, 0.3931, 0.5116]   )
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_xlarge_22k_1k_224_ema.pth":
            SCREAMING_SNAKE_CASE_				:      Any					=	torch.tensor([-0.6677, -0.1873, -0.8379]   )
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_xlarge_22k_1k_384_ema.pth":
            SCREAMING_SNAKE_CASE_				:      Optional[int]					=	torch.tensor([-0.7749, -0.2967, -0.6444]   )
      else:
            raise ValueError(F"Unknown URL: {checkpoint_url}"   )

      assert torch.allclose(logits[0, :3]  ,  _lowerCAmelCase  ,  atol=1E-3   )
      assert logits.shape == expected_shape

      Path(_lowerCAmelCase   ).mkdir(exist_ok=_lowerCAmelCase   )
      print(F"Saving model to {pytorch_dump_folder_path}"   )
      model.save_pretrained(_lowerCAmelCase   )
      print(F"Saving image processor to {pytorch_dump_folder_path}"   )
      image_processor.save_pretrained(_lowerCAmelCase   )

      print("Pushing model to the hub..."   )
      SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	"convnext"
      if "tiny" in checkpoint_url:
            model_name += "-tiny"
      elif "small" in checkpoint_url:
            model_name += "-small"
      elif "base" in checkpoint_url:
            model_name += "-base"
      elif "xlarge" in checkpoint_url:
            model_name += "-xlarge"
      elif "large" in checkpoint_url:
            model_name += "-large"
      if "224" in checkpoint_url:
            model_name += "-224"
      elif "384" in checkpoint_url:
            model_name += "-384"
      if "22k" in checkpoint_url and "1k" not in checkpoint_url:
            model_name += "-22k"
      if "22k" in checkpoint_url and "1k" in checkpoint_url:
            model_name += "-22k-1k"

      model.push_to_hub(
          repo_path_or_name=Path(_lowerCAmelCase  ,  _lowerCAmelCase   )  ,  organization="nielsr"  ,  commit_message="Add model"  ,  )


if __name__ == "__main__":
      snake_case_					  =	argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          '--checkpoint_url',
          default='https://dl.fbaipublicfiles.com/convnext/convnext_tiny_1k_224_ema.pth',
          type=str,
          help='URL of the original ConvNeXT checkpoint you\'d like to convert.',
      )
      parser.add_argument(
          '--pytorch_dump_folder_path',
          default=None,
          type=str,
          required=True,
          help='Path to the output PyTorch model directory.',
      )

      snake_case_					  =	parser.parse_args()
      convert_convnext_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path)




'''simple docstring'''






import json
import os
import unittest

from transformers import HerbertTokenizer, HerbertTokenizerFast
from transformers.models.herbert.tokenization_herbert import VOCAB_FILES_NAMES
from transformers.testing_utils import get_tests_dir, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class      SCREAMING_SNAKE_CASE__      (    _UpperCAmelCase,unittest.TestCase				):
    _A         =							HerbertTokenizer
    _A         =							HerbertTokenizerFast
    _A         =							True




    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          super().setUp()

          # Use a simpler test file without japanese/chinese characters
          with open(F"{get_tests_dir()}/fixtures/sample_text_no_unicode.txt"    ,					encoding="utf-8"      ) as f_data:
                SCREAMING_SNAKE_CASE_				:      List[str]					=	f_data.read().replace("\n\n"    ,					"\n"      ).strip()

          SCREAMING_SNAKE_CASE_				:      Dict					=	[
              "<s>",
              "</s>",
              "l",
              "o",
              "w",
              "e",
              "r",
              "s",
              "t",
              "i",
              "d",
              "n",
              "w</w>",
              "r</w>",
              "t</w>",
              "lo",
              "low",
              "er</w>",
              "low</w>",
              "lowest</w>",
              "newer</w>",
              "wider</w>",
              ",</w>",
              "<unk>",
          ]
          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	dict(zip(lowercase__    ,					range(len(lowercase__      )      )      )      )
          SCREAMING_SNAKE_CASE_				:      Dict					=	["l o 123", "lo w 1456", "e r</w> 1789", ""]

          SCREAMING_SNAKE_CASE_				:      Any					=	os.path.join(self.tmpdirname    ,					VOCAB_FILES_NAMES["vocab_file"]      )
          SCREAMING_SNAKE_CASE_				:      List[Any]					=	os.path.join(self.tmpdirname    ,					VOCAB_FILES_NAMES["merges_file"]      )
          with open(self.vocab_file    ,					"w"      ) as fp:
                fp.write(json.dumps(lowercase__      )      )
          with open(self.merges_file    ,					"w"      ) as fp:
                fp.write("\n".join(lowercase__      )      )




    def   __lowerCamelCase				( self    ,					lowercase__      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	"lower newer"
          SCREAMING_SNAKE_CASE_				:      str					=	"lower newer"
          return input_text, output_text




    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	self.tokenizer_class(vocab_file=self.vocab_file    ,					merges_file=self.merges_file      )

          SCREAMING_SNAKE_CASE_				:      int					=	"lower"
          SCREAMING_SNAKE_CASE_				:      List[str]					=	["low", "er</w>"]
          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	tokenizer.tokenize(lowercase__      )
          self.assertListEqual(lowercase__    ,					lowercase__      )

          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	tokens + ["<unk>"]
          SCREAMING_SNAKE_CASE_				:      Dict					=	[16, 17, 23]
          self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase__      )    ,					lowercase__      )




    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          if not self.test_rust_tokenizer:
                return

          SCREAMING_SNAKE_CASE_				:      Dict					=	self.get_tokenizer()
          SCREAMING_SNAKE_CASE_				:      Any					=	self.get_rust_tokenizer()

          SCREAMING_SNAKE_CASE_				:      List[Any]					=	"lower,newer"

          SCREAMING_SNAKE_CASE_				:      int					=	tokenizer.tokenize(lowercase__      )
          SCREAMING_SNAKE_CASE_				:      List[Any]					=	rust_tokenizer.tokenize(lowercase__      )
          self.assertListEqual(lowercase__    ,					lowercase__      )

          SCREAMING_SNAKE_CASE_				:      List[Any]					=	tokenizer.encode(lowercase__    ,					add_special_tokens=lowercase__      )
          SCREAMING_SNAKE_CASE_				:      int					=	rust_tokenizer.encode(lowercase__    ,					add_special_tokens=lowercase__      )
          self.assertListEqual(lowercase__    ,					lowercase__      )

          SCREAMING_SNAKE_CASE_				:      Any					=	self.get_rust_tokenizer()
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	tokenizer.encode(lowercase__      )
          SCREAMING_SNAKE_CASE_				:      Dict					=	rust_tokenizer.encode(lowercase__      )
          self.assertListEqual(lowercase__    ,					lowercase__      )







    @slow
    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      int					=	self.tokenizer_class.from_pretrained("allegro/herbert-base-cased"      )

          SCREAMING_SNAKE_CASE_				:      List[Any]					=	tokenizer.encode("konstruowanie sekwencji"    ,					add_special_tokens=lowercase__      )
          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	tokenizer.encode("konstruowanie wielu sekwencji"    ,					add_special_tokens=lowercase__      )

          SCREAMING_SNAKE_CASE_				:      List[str]					=	tokenizer.build_inputs_with_special_tokens(lowercase__      )
          SCREAMING_SNAKE_CASE_				:      Dict					=	tokenizer.build_inputs_with_special_tokens(lowercase__    ,					lowercase__      )

          assert encoded_sentence == [0] + text + [2]
          assert encoded_pair == [0] + text + [2] + text_a + [2]

