

# Usage:
# ./gen-card-allenai-wmt19.py

import os
from pathlib import Path







def _a			(	SCREAMING_SNAKE_CASE		:      Union[str, Any]   , SCREAMING_SNAKE_CASE		:      List[Any]   , SCREAMING_SNAKE_CASE		:      Optional[Any]   , SCREAMING_SNAKE_CASE		:      List[str]		)   ->   List[str]:

 """simple docstring"""






 __lowerCAmelCase:       Dict            =				{
     'en': 'Machine learning is great, isn\'t it?',
     'ru': 'Машинное обучение - это здорово, не так ли?',
     'de': 'Maschinelles Lernen ist großartig, nicht wahr?',
 }

 # BLUE scores as follows:
 # "pair": [fairseq, transformers]
 __lowerCAmelCase:       Tuple            =				{
     'wmt19-de-en-6-6-base': [0, 3_8.3_7],
     'wmt19-de-en-6-6-big': [0, 3_9.9_0],
 }
 __lowerCAmelCase:       int            =				f'''{src_lang}-{tgt_lang}'''

 __lowerCAmelCase:       Optional[int]            =				f'''
---

language:
- {src_lang}
- {tgt_lang}
thumbnail:
tags:
- translation
- wmt19
- allenai
license: apache-2.0
datasets:
- wmt19
metrics:
- bleu
---

# FSMT

## Model description

This is a ported version of fairseq-based [wmt19 transformer](https://github.com/jungokasai/deep-shallow/) for {src_lang}-{tgt_lang}.

For more details, please, see [Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation](https://arxiv.org/abs/2006.10369).

2 models are available:

* [wmt19-de-en-6-6-big](https://huggingface.co/allenai/wmt19-de-en-6-6-big)
* [wmt19-de-en-6-6-base](https://huggingface.co/allenai/wmt19-de-en-6-6-base)


## Intended uses & limitations

#### How to use

```python
from transformers import FSMTForConditionalGeneration, FSMTTokenizer
mname = "allenai/{model_name}"
tokenizer = FSMTTokenizer.from_pretrained(mname)
model = FSMTForConditionalGeneration.from_pretrained(mname)

input = "{texts[src_lang]}"
input_ids = tokenizer.encode(input, return_tensors="pt")
outputs = model.generate(input_ids)
decoded = tokenizer.decode(outputs[0], skip_special_tokens=True)
print(decoded) # {texts[tgt_lang]}

```

#### Limitations and bias


## Training data

Pretrained weights were left identical to the original model released by allenai. For more details, please, see the [paper](https://arxiv.org/abs/2006.10369).

## Eval results

Here are the BLEU scores:

model   |  transformers
-------|---------
{model_name}  |  {scores[model_name][1]}

The score was calculated using this code:

```bash
git clone https://github.com/huggingface/transformers
cd transformers
export PAIR={pair}
export DATA_DIR=data/$PAIR
export SAVE_DIR=data/$PAIR
export BS=8
export NUM_BEAMS=5
mkdir -p $DATA_DIR
sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
echo $PAIR
PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py allenai/{model_name} $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
```

## Data Sources

- [training, etc.](http://www.statmt.org/wmt19/)
- [test set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561)


### BibTeX entry and citation info

```
@misc{{kasai2020deep,
    title={{Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation}},
    author={{Jungo Kasai and Nikolaos Pappas and Hao Peng and James Cross and Noah A. Smith}},
    year={{2020}},
    eprint={{2006.10369}},
    archivePrefix={{arXiv}},
    primaryClass={{cs.CL}}
}}
```

'''
 model_card_dir.mkdir(parents=SCREAMING_SNAKE_CASE   , exist_ok=SCREAMING_SNAKE_CASE		)
 __lowerCAmelCase:       Optional[Any]            =				os.path.join(SCREAMING_SNAKE_CASE   , 'README.md'		)
 print(f'''Generating {path}'''		)
 with open(SCREAMING_SNAKE_CASE   , 'w'   , encoding='utf-8'		) as f:
  f.write(SCREAMING_SNAKE_CASE		)


# make sure we are under the root of the project
_a			=       Path(__file__).resolve().parent.parent.parent
_a			=       repo_dir / '''model_cards'''

for model_name in ["wmt19-de-en-6-6-base", "wmt19-de-en-6-6-big"]:
  _a			=       model_cards_dir / '''allenai''' / model_name
  write_model_card(model_card_dir, src_lang='''de''', tgt_lang='''en''', model_name=model_name)








import argparse
import datetime
import json
import time
import warnings
from logging import getLogger
from pathlib import Path
from typing import Dict, List

import torch
from tqdm import tqdm

from transformers import AutoModelForSeqaSeqLM, AutoTokenizer
from utils import calculate_bleu, calculate_rouge, chunks, parse_numeric_n_bool_cl_kwargs, use_task_specific_params


_a			=       getLogger(__name__)


_a			=       '''cuda''' if torch.cuda.is_available() else '''cpu'''







def _a			(	SCREAMING_SNAKE_CASE		:      List[str]   , SCREAMING_SNAKE_CASE		:      str   , SCREAMING_SNAKE_CASE		:      str   , SCREAMING_SNAKE_CASE		:      int = 8   , SCREAMING_SNAKE_CASE		:      str = DEFAULT_DEVICE   , SCREAMING_SNAKE_CASE		:      Union[str, Any]=False   , SCREAMING_SNAKE_CASE		:      Tuple="summarization"   , SCREAMING_SNAKE_CASE		:      int=None   , **SCREAMING_SNAKE_CASE		:      Optional[Any]   , )   ->   Dict:

 """simple docstring"""






 __lowerCAmelCase:       Union[str, Any]            =				Path(SCREAMING_SNAKE_CASE		).open('w'   , encoding='utf-8'		)
 __lowerCAmelCase:       Tuple            =				str(SCREAMING_SNAKE_CASE		)
 __lowerCAmelCase:       Optional[int]            =				AutoModelForSeqaSeqLM.from_pretrained(SCREAMING_SNAKE_CASE		).to(SCREAMING_SNAKE_CASE		)
 if fpaa:
  __lowerCAmelCase:       List[Any]            =				model.half()

 __lowerCAmelCase:       Dict            =				AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE		)
 logger.info(f'''Inferred tokenizer type: {tokenizer.__class__}'''		)  # if this is wrong, check config.model_type.

 __lowerCAmelCase:       Any            =				time.time()
 # update config with task specific params
 use_task_specific_params(SCREAMING_SNAKE_CASE   , SCREAMING_SNAKE_CASE		)
 if prefix is None:
  __lowerCAmelCase:       Optional[Any]            =				prefix or getattr(model.config   , 'prefix'   , ''		) or ''
 for examples_chunk in tqdm(list(chunks(SCREAMING_SNAKE_CASE   , SCREAMING_SNAKE_CASE		)		)		):
  __lowerCAmelCase:       str            =				[prefix + text for text in examples_chunk]
  __lowerCAmelCase:       Optional[Any]            =				tokenizer(SCREAMING_SNAKE_CASE   , return_tensors='pt'   , truncation=SCREAMING_SNAKE_CASE   , padding='longest'		).to(SCREAMING_SNAKE_CASE		)
  __lowerCAmelCase:       List[str]            =				model.generate(
      input_ids=batch.input_ids   , attention_mask=batch.attention_mask   , **SCREAMING_SNAKE_CASE   , )
  __lowerCAmelCase:       Dict            =				tokenizer.batch_decode(SCREAMING_SNAKE_CASE   , skip_special_tokens=SCREAMING_SNAKE_CASE   , clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE		)
  for hypothesis in dec:
   fout.write(hypothesis + '\n'		)
   fout.flush()
 fout.close()
 __lowerCAmelCase:       str            =				int(time.time() - start_time		)  # seconds
 __lowerCAmelCase:       Tuple            =				len(SCREAMING_SNAKE_CASE		)
 return {"n_obs": n_obs, "runtime": runtime, "seconds_per_sample": round(runtime / n_obs   , 4		)}







def _a			(	)   ->   Dict:

 """simple docstring"""






 return datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'		)







def _a			(	SCREAMING_SNAKE_CASE		:      Tuple=True		)   ->   int:

 """simple docstring"""






 __lowerCAmelCase:       str            =				argparse.ArgumentParser()
 parser.add_argument('model_name'   , type=SCREAMING_SNAKE_CASE   , help='like facebook/bart-large-cnn,t5-base, etc.'		)
 parser.add_argument('input_path'   , type=SCREAMING_SNAKE_CASE   , help='like cnn_dm/test.source'		)
 parser.add_argument('save_path'   , type=SCREAMING_SNAKE_CASE   , help='where to save summaries'		)
 parser.add_argument('--reference_path'   , type=SCREAMING_SNAKE_CASE   , required=SCREAMING_SNAKE_CASE   , help='like cnn_dm/test.target'		)
 parser.add_argument('--score_path'   , type=SCREAMING_SNAKE_CASE   , required=SCREAMING_SNAKE_CASE   , default='metrics.json'   , help='where to save metrics'		)
 parser.add_argument('--device'   , type=SCREAMING_SNAKE_CASE   , required=SCREAMING_SNAKE_CASE   , default=SCREAMING_SNAKE_CASE   , help='cuda, cuda:1, cpu etc.'		)
 parser.add_argument(
     '--prefix'   , type=SCREAMING_SNAKE_CASE   , required=SCREAMING_SNAKE_CASE   , default=SCREAMING_SNAKE_CASE   , help='will be added to the begininng of src examples'		)
 parser.add_argument('--task'   , type=SCREAMING_SNAKE_CASE   , default='summarization'   , help='used for task_specific_params + metrics'		)
 parser.add_argument('--bs'   , type=SCREAMING_SNAKE_CASE   , default=8   , required=SCREAMING_SNAKE_CASE   , help='batch size'		)
 parser.add_argument(
     '--n_obs'   , type=SCREAMING_SNAKE_CASE   , default=-1   , required=SCREAMING_SNAKE_CASE   , help='How many observations. Defaults to all.'		)
 parser.add_argument('--fp16'   , action='store_true'		)
 parser.add_argument('--dump-args'   , action='store_true'   , help='print the custom hparams with the results'		)
 parser.add_argument(
     '--info'   , nargs='?'   , type=SCREAMING_SNAKE_CASE   , const=datetime_now()   , help=(
         'use in conjunction w/ --dump-args to print with the results whatever other info you\'d like, e.g.'
         ' lang=en-ru. If no value is passed, the current datetime string will be used.'
     )   , )
 # Unspecified args like --num_beams=2 --decoder_start_token_id=4 are passed to model.generate
 __lowerCAmelCase     ,							__lowerCAmelCase:       Union[str, Any]            =				parser.parse_known_args()
 __lowerCAmelCase:       Union[str, Any]            =				parse_numeric_n_bool_cl_kwargs(SCREAMING_SNAKE_CASE		)
 if parsed_args and verbose:
  print(f'''parsed the following generate kwargs: {parsed_args}'''		)
 with open(args.input_path		) as f:
  __lowerCAmelCase:       Optional[int]            =				[' ' + x.rstrip() if 't5' in args.model_name else x.rstrip() for x in f.readlines()]
 if args.n_obs > 0:
  __lowerCAmelCase:       Tuple            =				examples[: args.n_obs]
 Path(args.save_path		).parent.mkdir(exist_ok=SCREAMING_SNAKE_CASE		)
 if args.reference_path is None and Path(args.score_path		).exists():
  warnings.warn(f'''score_path {args.score_path} will be overwritten unless you type ctrl-c.'''		)
 __lowerCAmelCase:       Tuple            =				generate_summaries_or_translations(
     SCREAMING_SNAKE_CASE   , args.save_path   , args.model_name   , batch_size=args.bs   , device=args.device   , fpaa=args.fpaa   , task=args.task   , prefix=args.prefix   , **SCREAMING_SNAKE_CASE   , )

 if args.reference_path is None:
  return {}

 # Compute scores
 __lowerCAmelCase:       Dict            =				calculate_bleu if 'translation' in args.task else calculate_rouge
 __lowerCAmelCase:       Optional[Any]            =				[x.rstrip() for x in open(args.save_path		).readlines()]
 __lowerCAmelCase:       Union[str, Any]            =				[x.rstrip() for x in open(args.reference_path		).readlines()][: len(SCREAMING_SNAKE_CASE		)]
 __lowerCAmelCase:       dict            =				score_fn(SCREAMING_SNAKE_CASE   , SCREAMING_SNAKE_CASE		)
 scores.update(SCREAMING_SNAKE_CASE		)

 if args.dump_args:
  scores.update(SCREAMING_SNAKE_CASE		)
 if args.info:
  __lowerCAmelCase:       Any            =				args.info

 if verbose:
  print(SCREAMING_SNAKE_CASE		)

 if args.score_path is not None:
  json.dump(SCREAMING_SNAKE_CASE   , open(args.score_path   , 'w'		)		)

 return scores


if __name__ == "__main__":
  # Usage for MT:
  # python run_eval.py MODEL_NAME $DATA_DIR/test.source $save_dir/test_translations.txt --reference_path $DATA_DIR/test.target --score_path $save_dir/test_bleu.json  --task translation $@
  run_generate(verbose=True)





