finetuning_llama.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sentencepiece as spm\n",
    "\n",
    "sp = spm.SentencePieceProcessor()\n",
    "sp.Load('bangla_char.model')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = \"আমার বাংলা ভাষা আমার গর্ব\"\n",
    "tokens = sp.encode_as_ids(text)\n",
    "print(tokens)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from huggingface_hub import notebook_login\n",
    "notebook_login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import wandb\n",
    "\n",
    "# Login to Weights & Biases\n",
    "wandb.login(key='WANDBKEY')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"max_split_size_mb:128\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import wandb\n",
    "from transformers import AutoModelForCausalLM, Trainer, TrainingArguments\n",
    "import sentencepiece as spm\n",
    "import torch\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from tqdm import tqdm\n",
    "from transformers import DataCollatorForLanguageModeling\n",
    "from custom_tokenizer import CustomBengaliTokenizer \n",
    "\n",
    "wandb.init(project=\"Transtokenizers\")\n",
    "\n",
    "# custom tokenizer\n",
    "tokenizer = CustomBengaliTokenizer(vocab_file=\"bangla_char.vocab\", model_file=\"bangla_char.model\")\n",
    "\n",
    "def tokenize_text_file(file_path, tokenizer):\n",
    "    with open(file_path, 'r', encoding='utf-8') as f:\n",
    "        lines = f.readlines()\n",
    "    tokenized_lines = [tokenizer.encode(line.strip()) for line in lines if line.strip()]\n",
    "    return tokenized_lines\n",
    "\n",
    "file_path = 'final_cleaned_bangla_corpus.txt'\n",
    "tokenized_data = tokenize_text_file(file_path, tokenizer)\n",
    "\n",
    "class CustomDataset(Dataset):\n",
    "    def __init__(self, tokenized_data):\n",
    "        self.tokenized_data = tokenized_data\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.tokenized_data)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        input_ids = torch.tensor(self.tokenized_data[idx], dtype=torch.long)\n",
    "        return {'input_ids': input_ids, 'labels': input_ids.clone()}\n",
    "\n",
    "dataset = CustomDataset(tokenized_data)\n",
    "train_dataloader = DataLoader(dataset, batch_size=8, shuffle=True, num_workers=4) \n",
    "\n",
    "model = AutoModelForCausalLM.from_pretrained(\"meta-llama/Meta-Llama-3.1-8B`\")\n",
    "\n",
    "data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "    output_dir='./results',\n",
    "    num_train_epochs=3,\n",
    "    per_device_train_batch_size=8,  \n",
    "    learning_rate=5e-5,\n",
    "    warmup_steps=500,\n",
    "    weight_decay=0.01,\n",
    "    logging_dir='./logs',\n",
    "    logging_steps=10,\n",
    "    save_steps=10_000,\n",
    "    save_total_limit=2,\n",
    "    gradient_accumulation_steps=4, \n",
    "    evaluation_strategy=\"steps\",\n",
    "    eval_steps=1000,\n",
    "    fp16=True,\n",
    "    dataloader_num_workers=4,\n",
    "    report_to=\"wandb\",\n",
    "    run_name=\"Transtokenizers_run\"\n",
    ")\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=dataset,\n",
    "    data_collator=data_collator\n",
    ")\n",
    "\n",
    "trainer.train()\n",
    "\n",
    "wandb.finish()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "from unsloth import FastLanguageModel\n",
    "import torch\n",
    "from datasets import Dataset\n",
    "import sentencepiece as spm\n",
    "from tqdm import tqdm\n",
    "import wandb\n",
    "\n",
    "\n",
    "wandb.init(project=\"Transtokenizers\")\n",
    "\n",
    "\n",
    "def load_and_tokenize_text(file_path, sp_model):\n",
    "    sp = spm.SentencePieceProcessor(model_file=sp_model)\n",
    "    with open(file_path, 'r', encoding='utf-8') as f:\n",
    "        lines = f.readlines()\n",
    "    tokenized_lines = [sp.encode(line.strip(), out_type=int) for line in tqdm(lines, desc=\"Tokenizing lines\") if line.strip()]\n",
    "    return tokenized_lines\n",
    "\n",
    "\n",
    "file_path = 'final_cleaned_bangla_corpus.txt'\n",
    "sp_model = 'bangla_bpe.model'\n",
    "tokenized_data = load_and_tokenize_text(file_path, sp_model)\n",
    "\n",
    "\n",
    "def create_hf_dataset(tokenized_data):\n",
    "    texts = [\" \".join(map(str, tokens)) for tokens in tokenized_data]\n",
    "    dataset = Dataset.from_dict({\"text\": texts})\n",
    "    return dataset\n",
    "\n",
    "dataset = create_hf_dataset(tokenized_data)\n",
    "\n",
    "\n",
    "model_name = \"unsloth/Meta-Llama-3.1-8B\"\n",
    "\n",
    "\n",
    "model, tokenizer = FastLanguageModel.from_pretrained(\n",
    "    model_name=model_name,\n",
    "    max_seq_length=2048,\n",
    "    dtype=None,  \n",
    "    load_in_4bit=True,  \n",
    ")\n",
    "\n",
    "model = FastLanguageModel.get_peft_model(\n",
    "    model,\n",
    "    r=16,\n",
    "    target_modules=[\"q_proj\", \"k_proj\", \"v_proj\", \"o_proj\", \"gate_proj\", \"up_proj\", \"down_proj\"],\n",
    "    lora_alpha=16,\n",
    "    lora_dropout=0,\n",
    "    bias=\"none\",\n",
    "    use_gradient_checkpointing=\"unsloth\",\n",
    "    random_state=3407,\n",
    ")\n",
    "\n",
    "def format_data(examples):\n",
    "    texts = examples[\"text\"]\n",
    "    formatted_texts = [text + tokenizer.eos_token for text in texts]\n",
    "    return {\"text\": formatted_texts}\n",
    "\n",
    "dataset = dataset.map(format_data, batched=True, desc=\"Formatting data\")\n",
    "\n",
    "from trl import SFTTrainer\n",
    "from transformers import TrainingArguments\n",
    "from unsloth import is_bfloat16_supported\n",
    "\n",
    "trainer = SFTTrainer(\n",
    "    model=model,\n",
    "    tokenizer=tokenizer,\n",
    "    train_dataset=dataset,\n",
    "    dataset_text_field=\"text\",\n",
    "    max_seq_length=2048,\n",
    "    dataset_num_proc=2,\n",
    "    packing=False,\n",
    "    args=TrainingArguments(\n",
    "        per_device_train_batch_size=2,\n",
    "        gradient_accumulation_steps=4,\n",
    "        warmup_steps=5,\n",
    "        max_steps=60,  \n",
    "        learning_rate=2e-4,\n",
    "        fp16=not is_bfloat16_supported(),\n",
    "        bf16=is_bfloat16_supported(),\n",
    "        logging_steps=1,\n",
    "        optim=\"adamw_8bit\",\n",
    "        weight_decay=0.01,\n",
    "        lr_scheduler_type=\"linear\",\n",
    "        seed=3407,\n",
    "        output_dir=\"outputs\",\n",
    "        report_to=\"wandb\",\n",
    "        run_name=\"Transtokenizers_run\"\n",
    "    ),\n",
    ")\n",
    "\n",
    "\n",
    "trainer_stats = trainer.train()\n",
    "\n",
    "\n",
    "gpu_stats = torch.cuda.get_device_properties(0)\n",
    "start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)\n",
    "max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3)\n",
    "used_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)\n",
    "used_memory_for_lora = round(used_memory - start_gpu_memory, 3)\n",
    "used_percentage = round(used_memory / max_memory * 100, 3)\n",
    "lora_percentage = round(used_memory_for_lora / max_memory * 100, 3)\n",
    "\n",
    "print(f\"Training runtime: {trainer_stats.metrics['train_runtime']} seconds\")\n",
    "print(f\"Training runtime in minutes: {round(trainer_stats.metrics['train_runtime'] / 60, 2)} minutes\")\n",
    "print(f\"Peak reserved memory: {used_memory} GB\")\n",
    "print(f\"Peak reserved memory for training: {used_memory_for_lora} GB\")\n",
    "print(f\"Peak reserved memory % of max memory: {used_percentage} %\")\n",
    "print(f\"Peak reserved memory for training % of max memory: {lora_percentage} %\")\n",
    "\n",
    "\n",
    "model.save_pretrained(\"lora_model\")\n",
    "tokenizer.save_pretrained(\"lora_model\")\n",
    "\n",
    "\n",
    "wandb.finish()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import requests\n",
    "\n",
    "def download_file(url, filename):\n",
    "    response = requests.get(url)\n",
    "    response.raise_for_status()  \n",
    "    with open(filename, 'w', encoding='utf-8') as file:\n",
    "        file.write(response.text)\n",
    "\n",
    "ben_url = 'https://raw.githubusercontent.com/BibleNLP/ebible/main/corpus/ben-ben2017.txt'\n",
    "eng_url = 'https://raw.githubusercontent.com/BibleNLP/ebible/main/corpus/eng-enggnv.txt'\n",
    "\n",
    "\n",
    "download_file(ben_url, 'ben-ben2017.txt')\n",
    "download_file(eng_url, 'eng-enggnv.txt')\n",
    "\n",
    "\n",
    "\n",
    "import sentencepiece as spm\n",
    "\n",
    "\n",
    "corpus_file = 'eng-enggnv.txt'\n",
    "model_prefix = 'english_bpe'\n",
    "vocab_size = 32000\n",
    "\n",
    "spm.SentencePieceTrainer.train(\n",
    "    input=corpus_file,\n",
    "    model_prefix='english_bpe',\n",
    "    vocab_size=32000,\n",
    "    model_type='bpe',\n",
    "    character_coverage=0.9995,\n",
    "    max_sentence_length=10000\n",
    ")\n",
    "\n",
    "print(f\"Tokenizer trained and saved as {model_prefix}.model and {model_prefix}.vocab\")\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "sentencepiece_trainer.cc(78) LOG(INFO) Starts training with : \n",
      "trainer_spec {\n",
      "  input: eng-enggnv.txt\n",
      "  input_format: \n",
      "  model_prefix: english_bpe\n",
      "  model_type: BPE\n",
      "  vocab_size: 32000\n",
      "  self_test_sample_size: 0\n",
      "  character_coverage: 0.9995\n",
      "  input_sentence_size: 0\n",
      "  shuffle_input_sentence: 1\n",
      "  seed_sentencepiece_size: 1000000\n",
      "  shrinking_factor: 0.75\n",
      "  max_sentence_length: 10000\n",
      "  num_threads: 16\n",
      "  num_sub_iterations: 2\n",
      "  max_sentencepiece_length: 16\n",
      "  split_by_unicode_script: 1\n",
      "  split_by_number: 1\n",
      "  split_by_whitespace: 1\n",
      "  split_digits: 0\n",
      "  pretokenization_delimiter: \n",
      "  treat_whitespace_as_suffix: 0\n",
      "  allow_whitespace_only_pieces: 0\n",
      "  required_chars: \n",
      "  byte_fallback: 0\n",
      "  vocabulary_output_piece_score: 1\n",
      "  train_extremely_large_corpus: 0\n",
      "  seed_sentencepieces_file: \n",
      "  hard_vocab_limit: 1\n",
      "  use_all_vocab: 0\n",
      "  unk_id: 0\n",
      "  bos_id: 1\n",
      "  eos_id: 2\n",
      "  pad_id: -1\n",
      "  unk_piece: <unk>\n",
      "  bos_piece: <s>\n",
      "  eos_piece: </s>\n",
      "  pad_piece: <pad>\n",
      "  unk_surface:  ⁇ \n",
      "  enable_differential_privacy: 0\n",
      "  differential_privacy_noise_level: 0\n",
      "  differential_privacy_clipping_threshold: 0\n",
      "}\n",
      "normalizer_spec {\n",
      "  name: nmt_nfkc\n",
      "  add_dummy_prefix: 1\n",
      "  remove_extra_whitespaces: 1\n",
      "  escape_whitespaces: 1\n",
      "  normalization_rule_tsv: \n",
      "}\n",
      "denormalizer_spec {}\n",
      "trainer_interface.cc(353) LOG(INFO) SentenceIterator is not specified. Using MultiFileSentenceIterator.\n",
      "trainer_interface.cc(185) LOG(INFO) Loading corpus: eng-enggnv.txt\n",
      "trainer_interface.cc(409) LOG(INFO) Loaded all 30807 sentences\n",
      "trainer_interface.cc(425) LOG(INFO) Adding meta_piece: <unk>\n",
      "trainer_interface.cc(425) LOG(INFO) Adding meta_piece: <s>\n",
      "trainer_interface.cc(425) LOG(INFO) Adding meta_piece: </s>\n",
      "trainer_interface.cc(430) LOG(INFO) Normalizing sentences...\n",
      "trainer_interface.cc(539) LOG(INFO) all chars count=4184341\n",
      "trainer_interface.cc(550) LOG(INFO) Done: 99.9529% characters are covered.\n",
      "trainer_interface.cc(560) LOG(INFO) Alphabet size=52\n",
      "trainer_interface.cc(561) LOG(INFO) Final character coverage=0.999529\n",
      "trainer_interface.cc(592) LOG(INFO) Done! preprocessed 30807 sentences.\n",
      "trainer_interface.cc(598) LOG(INFO) Tokenizing input sentences with whitespace: 30807\n",
      "trainer_interface.cc(609) LOG(INFO) Done! 30096\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=150887 min_freq=1\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=26480 size=20 all=1472 active=1419 piece=▁m\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=14018 size=40 all=2159 active=2106 piece=it\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=10031 size=60 all=2973 active=2920 piece=ue\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=7171 size=80 all=3477 active=3424 piece=gh\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5709 size=100 all=4186 active=4133 piece=ent\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=5690 min_freq=234\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4503 size=120 all=4779 active=1541 piece=▁God\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=3772 size=140 all=5183 active=1945 piece=ro\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=3011 size=160 all=5860 active=2622 piece=▁Is\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=2604 size=180 all=6353 active=3115 piece=rael\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=2391 size=200 all=6840 active=3602 piece=▁C\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=2383 min_freq=246\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=2136 size=220 all=7409 active=1526 piece=ide\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=1896 size=240 all=7950 active=2067 piece=▁house\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=1716 size=260 all=8326 active=2443 piece=▁also\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=1570 size=280 all=8569 active=2686 piece=oy\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=1447 size=300 all=8910 active=3027 piece=▁tw\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=1442 min_freq=223\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=1270 size=320 all=9230 active=1312 piece=▁seru\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=1121 size=340 all=9682 active=1764 piece=ene\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=1055 size=360 all=9898 active=1980 piece=ned\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=973 size=380 all=10076 active=2158 piece=▁Iesus\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=918 size=400 all=10273 active=2355 piece=▁sin\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=916 min_freq=195\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=876 size=420 all=10555 active=1274 piece=asse\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=828 size=440 all=10711 active=1430 piece=▁(\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=789 size=460 all=10900 active=1619 piece=ning\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=744 size=480 all=11072 active=1791 piece=itt\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=711 size=500 all=11425 active=2144 piece=▁del\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=711 min_freq=173\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=664 size=520 all=11596 active=1161 piece=▁Ierusalem\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=619 size=540 all=11810 active=1375 piece=ward\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=597 size=560 all=12049 active=1614 piece=▁yet\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=568 size=580 all=12164 active=1729 piece=▁ro\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=526 size=600 all=12324 active=1889 piece=ret\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=526 min_freq=155\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=498 size=620 all=12502 active=1145 piece=▁glor\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=468 size=640 all=12749 active=1392 piece=euer\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=451 size=660 all=13014 active=1657 piece=orde\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=440 size=680 all=13088 active=1731 piece=▁Name\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=426 size=700 all=13206 active=1849 piece=▁Let\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=425 min_freq=141\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=410 size=720 all=13425 active=1219 piece=▁ent\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=395 size=740 all=13600 active=1394 piece=▁thir\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=378 size=760 all=13775 active=1569 piece=ched\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=368 size=780 all=13911 active=1705 piece=▁foure\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=357 size=800 all=14036 active=1830 piece=▁foorth\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=355 min_freq=123\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=347 size=820 all=14149 active=1113 piece=▁vnc\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=338 size=840 all=14268 active=1232 piece=▁In\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=326 size=860 all=14386 active=1350 piece=▁taken\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=317 size=880 all=14477 active=1441 piece=▁wilder\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=307 size=900 all=14599 active=1563 piece=▁gra\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=307 min_freq=112\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=299 size=920 all=14733 active=1123 piece=ified\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=292 size=940 all=14866 active=1256 piece=▁long\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=286 size=960 all=14997 active=1387 piece=▁written\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=277 size=980 all=15127 active=1517 piece=▁turne\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=268 size=1000 all=15169 active=1559 piece=▁Leuites\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=267 min_freq=102\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=259 size=1020 all=15336 active=1168 piece=▁cub\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=254 size=1040 all=15368 active=1200 piece=▁daughters\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=247 size=1060 all=15452 active=1284 piece=weene\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=241 size=1080 all=15499 active=1331 piece=▁tribe\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=234 size=1100 all=15599 active=1431 piece=▁cloth\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=234 min_freq=95\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=228 size=1120 all=15690 active=1087 piece=▁pit\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=223 size=1140 all=15784 active=1181 piece=oint\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=216 size=1160 all=16006 active=1403 piece=oc\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=208 size=1180 all=16175 active=1572 piece=▁some\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=203 size=1200 all=16249 active=1646 piece=▁Priests\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=202 min_freq=87\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=199 size=1220 all=16342 active=1094 piece=▁char\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=194 size=1240 all=16490 active=1242 piece=iles\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=190 size=1260 all=16571 active=1323 piece=▁All\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=185 size=1280 all=16696 active=1448 piece=▁stones\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=180 size=1300 all=16774 active=1526 piece=▁tou\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=180 min_freq=80\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=176 size=1320 all=16819 active=1043 piece=▁reigned\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=172 size=1340 all=16899 active=1123 piece=▁Elders\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=169 size=1360 all=16992 active=1216 piece=▁gold\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=166 size=1380 all=17109 active=1333 piece=▁kind\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=162 size=1400 all=17165 active=1389 piece=▁Of\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=162 min_freq=74\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=159 size=1420 all=17208 active=1044 piece=ased\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=156 size=1440 all=17305 active=1141 piece=med\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=154 size=1460 all=17408 active=1244 piece=▁slaine\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=150 size=1480 all=17512 active=1348 piece=▁pill\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=147 size=1500 all=17573 active=1409 piece=▁cam\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=147 min_freq=68\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=145 size=1520 all=17694 active=1117 piece=▁Ioab\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=143 size=1540 all=17742 active=1165 piece=▁treas\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=140 size=1560 all=17764 active=1187 piece=▁opened\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=137 size=1580 all=17798 active=1221 piece=alt\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=135 size=1600 all=17885 active=1308 piece=▁answere\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=135 min_freq=64\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=131 size=1620 all=17972 active=1086 piece=▁Es\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=129 size=1640 all=18023 active=1137 piece=▁meete\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=126 size=1660 all=18090 active=1204 piece=▁mes\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=122 size=1680 all=18128 active=1242 piece=bu\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=120 size=1700 all=18196 active=1310 piece=▁giueth\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=120 min_freq=60\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=117 size=1720 all=18244 active=1049 piece=▁hat\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=115 size=1740 all=18319 active=1124 piece=aues\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=113 size=1760 all=18363 active=1168 piece=erue\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=112 size=1780 all=18427 active=1232 piece=▁daies\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=110 size=1800 all=18498 active=1303 piece=▁ang\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=110 min_freq=56\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=109 size=1820 all=18526 active=1023 piece=▁tribes\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=107 size=1840 all=18632 active=1129 piece=▁doest\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=105 size=1860 all=18694 active=1191 piece=▁foote\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=103 size=1880 all=18779 active=1276 piece=▁famine\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=101 size=1900 all=18848 active=1345 piece=▁afraide\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=101 min_freq=53\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=99 size=1920 all=18914 active=1067 piece=▁last\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=97 size=1940 all=18977 active=1130 piece=ma\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=96 size=1960 all=19076 active=1229 piece=▁small\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=94 size=1980 all=19147 active=1300 piece=▁drun\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=93 size=2000 all=19165 active=1318 piece=▁sicke\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=93 min_freq=50\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=91 size=2020 all=19216 active=1051 piece=▁grie\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=90 size=2040 all=19254 active=1089 piece=▁likewise\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=88 size=2060 all=19296 active=1131 piece=▁serued\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=86 size=2080 all=19365 active=1200 piece=▁Her\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=85 size=2100 all=19397 active=1232 piece=▁taught\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=85 min_freq=48\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=84 size=2120 all=19503 active=1107 piece=▁rocke\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=82 size=2140 all=19530 active=1134 piece=aut\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=81 size=2160 all=19582 active=1186 piece=▁nine\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=80 size=2180 all=19608 active=1212 piece=▁shep\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=78 size=2200 all=19628 active=1232 piece=rs\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=78 min_freq=45\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=77 size=2220 all=19690 active=1052 piece=ins\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=76 size=2240 all=19802 active=1164 piece=onde\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=76 size=2260 all=19851 active=1213 piece=▁knowest\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=74 size=2280 all=19894 active=1256 piece=ales\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=74 size=2300 all=19907 active=1269 piece=▁compassion\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=74 min_freq=43\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=72 size=2320 all=19951 active=1044 piece=▁path\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=71 size=2340 all=19991 active=1084 piece=▁cried\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=70 size=2360 all=20018 active=1111 piece=▁assembled\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=68 size=2380 all=20085 active=1178 piece=ok\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=68 size=2400 all=20173 active=1266 piece=▁beloued\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=68 min_freq=40\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=67 size=2420 all=20225 active=1061 piece=▁Eliiah\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=66 size=2440 all=20289 active=1125 piece=▁doct\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=65 size=2460 all=20355 active=1191 piece=▁See\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=64 size=2480 all=20372 active=1208 piece=icho\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=64 size=2500 all=20393 active=1229 piece=▁Scribes\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=64 min_freq=38\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=63 size=2520 all=20405 active=1032 piece=▁images\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=62 size=2540 all=20438 active=1065 piece=▁solde\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=61 size=2560 all=20474 active=1101 piece=▁edge\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=60 size=2580 all=20510 active=1137 piece=▁Haz\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=60 size=2600 all=20536 active=1163 piece=▁foundation\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=60 min_freq=36\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=59 size=2620 all=20584 active=1075 piece=▁bitter\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=58 size=2640 all=20628 active=1119 piece=▁linen\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=57 size=2660 all=20656 active=1147 piece=▁Sat\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=56 size=2680 all=20685 active=1176 piece=▁Lo\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=56 size=2700 all=20740 active=1231 piece=▁desired\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=56 min_freq=34\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=55 size=2720 all=20822 active=1119 piece=▁spot\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=55 size=2740 all=20828 active=1125 piece=▁kindled\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=54 size=2760 all=20880 active=1177 piece=▁faint\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=53 size=2780 all=20895 active=1192 piece=adok\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=53 size=2800 all=20924 active=1221 piece=▁garden\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=53 min_freq=32\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=52 size=2820 all=20960 active=1082 piece=▁crue\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=52 size=2840 all=20957 active=1079 piece=▁heareth\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=51 size=2860 all=21041 active=1163 piece=▁Laban\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=51 size=2880 all=21032 active=1154 piece=▁rebellious\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=50 size=2900 all=21094 active=1216 piece=▁imag\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=50 min_freq=31\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=50 size=2920 all=21088 active=1045 piece=▁signes\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=49 size=2940 all=21146 active=1103 piece=ible\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=49 size=2960 all=21191 active=1148 piece=▁milke\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=49 size=2980 all=21185 active=1142 piece=▁sinners\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=48 size=3000 all=21217 active=1174 piece=▁drawe\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=48 min_freq=29\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=47 size=3020 all=21247 active=1091 piece=▁ign\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=47 size=3040 all=21289 active=1133 piece=▁ignor\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=47 size=3060 all=21292 active=1136 piece=▁Remember\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=46 size=3080 all=21335 active=1179 piece=erued\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=46 size=3100 all=21334 active=1178 piece=▁seconde\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=46 min_freq=28\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=45 size=3120 all=21373 active=1106 piece=ideon\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=45 size=3140 all=21388 active=1121 piece=▁greene\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=44 size=3160 all=21437 active=1170 piece=aking\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=44 size=3180 all=21460 active=1193 piece=▁yeres\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=43 size=3200 all=21462 active=1195 piece=▁ol\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=43 min_freq=26\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=43 size=3220 all=21468 active=1078 piece=▁dominion\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=42 size=3240 all=21490 active=1100 piece=▁nat\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=42 size=3260 all=21511 active=1121 piece=▁Shimei\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=41 size=3280 all=21507 active=1117 piece=▁yn\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=41 size=3300 all=21578 active=1188 piece=▁Riuer\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=41 min_freq=25\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=41 size=3320 all=21569 active=1070 piece=▁woordes\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=40 size=3340 all=21645 active=1146 piece=▁kid\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=40 size=3360 all=21672 active=1173 piece=▁reconc\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=39 size=3380 all=21725 active=1226 piece=ting\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=39 size=3400 all=21759 active=1260 piece=▁Merari\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=39 min_freq=24\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=39 size=3420 all=21751 active=1080 piece=▁tabernacle\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=38 size=3440 all=21809 active=1138 piece=▁Marie\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=38 size=3460 all=21809 active=1138 piece=▁entring\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=37 size=3480 all=21861 active=1190 piece=▁hol\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=37 size=3500 all=21891 active=1220 piece=▁horne\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=37 min_freq=23\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=37 size=3520 all=21887 active=1090 piece=▁tongues\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=36 size=3540 all=21925 active=1128 piece=▁med\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=36 size=3560 all=21957 active=1160 piece=▁recom\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=35 size=3580 all=21984 active=1187 piece=▁Ebre\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=35 size=3600 all=21981 active=1184 piece=▁emptie\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=35 min_freq=22\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=35 size=3620 all=21975 active=1094 piece=▁standing\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=34 size=3640 all=22031 active=1150 piece=▁mad\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=34 size=3660 all=22057 active=1176 piece=▁molten\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=33 size=3680 all=22073 active=1192 piece=dle\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=33 size=3700 all=22160 active=1279 piece=ilous\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=33 min_freq=21\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=33 size=3720 all=22160 active=1106 piece=▁letter\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=33 size=3740 all=22144 active=1090 piece=▁destroied\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=32 size=3760 all=22206 active=1152 piece=onia\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=32 size=3780 all=22244 active=1190 piece=▁knees\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=32 size=3800 all=22237 active=1183 piece=▁members\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=32 min_freq=20\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=31 size=3820 all=22260 active=1135 piece=say\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=31 size=3840 all=22330 active=1205 piece=▁shad\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=31 size=3860 all=22335 active=1210 piece=▁Isaiah\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=31 size=3880 all=22330 active=1205 piece=▁besieged\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=30 size=3900 all=22337 active=1212 piece=▁Ex\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=30 min_freq=19\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=30 size=3920 all=22401 active=1177 piece=▁Elis\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=30 size=3940 all=22407 active=1183 piece=▁Ethiop\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=30 size=3960 all=22394 active=1170 piece=▁subiect\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=29 size=3980 all=22409 active=1185 piece=yeth\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=29 size=4000 all=22424 active=1200 piece=edonia\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=29 min_freq=18\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=29 size=4020 all=22420 active=1117 piece=▁sitting\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=29 size=4040 all=22407 active=1104 piece=▁strengthen\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=28 size=4060 all=22479 active=1176 piece=▁sic\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=28 size=4080 all=22497 active=1194 piece=▁seate\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=28 size=4100 all=22495 active=1192 piece=▁woorke\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=28 min_freq=18\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=27 size=4120 all=22492 active=1120 piece=ded\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=27 size=4140 all=22552 active=1180 piece=▁kine\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=27 size=4160 all=22556 active=1184 piece=▁whirle\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=27 size=4180 all=22548 active=1176 piece=▁mischiefe\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=26 size=4200 all=22614 active=1242 piece=shem\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=26 min_freq=17\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=26 size=4220 all=22653 active=1167 piece=▁store\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=26 size=4240 all=22647 active=1161 piece=▁grieued\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=26 size=4260 all=22635 active=1149 piece=▁perpetuall\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=25 size=4280 all=22683 active=1197 piece=▁Eun\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=25 size=4300 all=22742 active=1256 piece=▁drew\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=25 min_freq=16\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=25 size=4320 all=22751 active=1146 piece=▁contem\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=25 size=4340 all=22745 active=1140 piece=▁shepher\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=24 size=4360 all=22755 active=1150 piece=ahaz\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=24 size=4380 all=22810 active=1205 piece=▁eunu\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=24 size=4400 all=22828 active=1223 piece=▁seeme\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=24 min_freq=16\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=24 size=4420 all=22814 active=1128 piece=▁profit\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=24 size=4440 all=22817 active=1131 piece=▁bullocks\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=23 size=4460 all=22846 active=1160 piece=erd\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=23 size=4480 all=22906 active=1220 piece=ining\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=23 size=4500 all=22920 active=1234 piece=▁ended\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=23 min_freq=15\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=23 size=4520 all=22919 active=1145 piece=▁driuen\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=23 size=4540 all=22907 active=1133 piece=▁nombers\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=23 size=4560 all=22898 active=1124 piece=▁deliuerance\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=22 size=4580 all=22976 active=1202 piece=▁cun\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=22 size=4600 all=23002 active=1228 piece=▁Nabal\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=22 min_freq=15\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=22 size=4620 all=23001 active=1149 piece=▁Public\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=22 size=4640 all=22995 active=1143 piece=▁quickly\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=21 size=4660 all=22990 active=1138 piece=ley\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=21 size=4680 all=23036 active=1184 piece=entur\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=21 size=4700 all=23062 active=1210 piece=isheth\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=21 min_freq=14\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=21 size=4720 all=23071 active=1154 piece=▁memor\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=21 size=4740 all=23077 active=1160 piece=▁herbes\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=21 size=4760 all=23059 active=1142 piece=▁matters\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=21 size=4780 all=23044 active=1127 piece=▁hundreths\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=20 size=4800 all=23056 active=1139 piece=▁Ret\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=20 min_freq=14\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=20 size=4820 all=23077 active=1173 piece=habiah\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=20 size=4840 all=23082 active=1178 piece=▁token\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=20 size=4860 all=23077 active=1173 piece=▁speach\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=20 size=4880 all=23064 active=1160 piece=▁Abrahams\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=20 size=4900 all=23046 active=1142 piece=▁filthinesse\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=20 min_freq=13\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=19 size=4920 all=23088 active=1195 piece=itus\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=19 size=4940 all=23133 active=1240 piece=▁Ahim\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=19 size=4960 all=23155 active=1262 piece=▁Aholi\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=19 size=4980 all=23150 active=1257 piece=linesse\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=19 size=5000 all=23144 active=1251 piece=▁spring\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=19 min_freq=13\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=19 size=5020 all=23129 active=1140 piece=▁thunder\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=19 size=5040 all=23110 active=1121 piece=▁countreys\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=18 size=5060 all=23129 active=1140 piece=aire\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=18 size=5080 all=23196 active=1207 piece=▁der\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=18 size=5100 all=23229 active=1240 piece=▁maid\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=18 min_freq=13\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=18 size=5120 all=23241 active=1172 piece=▁zeale\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=18 size=5140 all=23223 active=1154 piece=▁neckes\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=18 size=5160 all=23215 active=1146 piece=▁fellowe\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=18 size=5180 all=23207 active=1138 piece=▁pouertie\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=18 size=5200 all=23189 active=1120 piece=▁Ahithophel\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=18 min_freq=12\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=17 size=5220 all=23229 active=1200 piece=gail\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=17 size=5240 all=23282 active=1253 piece=▁rid\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=17 size=5260 all=23295 active=1266 piece=▁daye\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=17 size=5280 all=23298 active=1269 piece=▁Sidon\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=17 size=5300 all=23301 active=1272 piece=▁tread\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=17 min_freq=12\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=17 size=5320 all=23293 active=1156 piece=▁spared\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=17 size=5340 all=23287 active=1150 piece=▁iudgeth\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=17 size=5360 all=23271 active=1134 piece=▁greatest\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=17 size=5380 all=23253 active=1116 piece=▁profitable\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=16 size=5400 all=23319 active=1182 piece=itub\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=16 min_freq=11\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=16 size=5420 all=23350 active=1196 piece=▁tel\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=16 size=5440 all=23372 active=1218 piece=▁affe\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=16 size=5460 all=23392 active=1238 piece=▁Hadad\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=16 size=5480 all=23386 active=1232 piece=▁Libnah\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=16 size=5500 all=23374 active=1220 piece=▁purged\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=16 min_freq=11\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=16 size=5520 all=23359 active=1154 piece=▁already\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=16 size=5540 all=23342 active=1137 piece=▁mysterie\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=16 size=5560 all=23332 active=1127 piece=▁fortresse\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=15 size=5580 all=23317 active=1112 piece=fo\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=15 size=5600 all=23397 active=1192 piece=arch\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=15 min_freq=11\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=15 size=5620 all=23456 active=1223 piece=▁Shu\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=15 size=5640 all=23494 active=1261 piece=▁Bapt\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=15 size=5660 all=23507 active=1274 piece=▁Adoni\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=15 size=5680 all=23511 active=1278 piece=▁holes\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=15 size=5700 all=23498 active=1265 piece=▁closed\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=15 min_freq=10\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=15 size=5720 all=23483 active=1160 piece=���Gospell\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=15 size=5740 all=23470 active=1147 piece=▁stripes\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=15 size=5760 all=23454 active=1131 piece=▁brimstone\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=14 size=5780 all=23449 active=1126 piece=wa\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=14 size=5800 all=23519 active=1196 piece=shua\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=14 min_freq=10\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=14 size=5820 all=23566 active=1222 piece=ately\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=14 size=5840 all=23597 active=1253 piece=▁shot\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=14 size=5860 all=23612 active=1268 piece=▁cloke\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=14 size=5880 all=23609 active=1265 piece=▁Hushai\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=14 size=5900 all=23591 active=1247 piece=▁hinder\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=14 min_freq=10\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=14 size=5920 all=23580 active=1166 piece=▁badgers\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=14 size=5940 all=23570 active=1156 piece=▁contrary\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=14 size=5960 all=23553 active=1139 piece=▁handmayde\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=14 size=5980 all=23536 active=1122 piece=▁congregation\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=13 size=6000 all=23582 active=1168 piece=aued\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=13 min_freq=9\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=13 size=6020 all=23635 active=1229 piece=▁vnw\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=13 size=6040 all=23665 active=1259 piece=▁Iddo\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=13 size=6060 all=23664 active=1258 piece=▁pass\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=13 size=6080 all=23686 active=1280 piece=▁Sheph\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=13 size=6100 all=23695 active=1289 piece=▁ruine\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=13 min_freq=9\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=13 size=6120 all=23685 active=1175 piece=▁bowles\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=13 size=6140 all=23675 active=1165 piece=▁Gadites\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=13 size=6160 all=23658 active=1148 piece=▁serueth\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=13 size=6180 all=23644 active=1134 piece=▁petition\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=13 size=6200 all=23624 active=1114 piece=▁Merciseate\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=13 min_freq=9\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=12 size=6220 all=23611 active=1169 piece=gai\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=12 size=6240 all=23677 active=1235 piece=mans\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=12 size=6260 all=23709 active=1267 piece=anias\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=12 size=6280 all=23737 active=1295 piece=▁dres\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=12 size=6300 all=23761 active=1319 piece=athite\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=12 min_freq=9\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=12 size=6320 all=23774 active=1196 piece=▁Shamm\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=12 size=6340 all=23770 active=1192 piece=▁maist\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=12 size=6360 all=23765 active=1187 piece=▁Pharez\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=12 size=6380 all=23754 active=1176 piece=▁ouerse\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=12 size=6400 all=23748 active=1170 piece=▁breasts\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=12 min_freq=8\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=12 size=6420 all=23738 active=1178 piece=▁saidest\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=12 size=6440 all=23721 active=1161 piece=▁pastures\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=12 size=6460 all=23701 active=1141 piece=▁brightnes\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=12 size=6480 all=23683 active=1123 piece=▁Shephatiah\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=11 size=6500 all=23685 active=1125 piece=ina\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=11 min_freq=8\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=11 size=6520 all=23742 active=1236 piece=anna\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=11 size=6540 all=23786 active=1280 piece=▁Fin\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=11 size=6560 all=23816 active=1310 piece=cesse\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=11 size=6580 all=23850 active=1344 piece=▁Geba\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=11 size=6600 all=23858 active=1352 piece=▁seue\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=11 min_freq=8\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=11 size=6620 all=23857 active=1190 piece=▁Reuel\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=11 size=6640 all=23855 active=1188 piece=▁grate\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=11 size=6660 all=23845 active=1178 piece=olished\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=11 size=6680 all=23837 active=1170 piece=▁camped\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=11 size=6700 all=23818 active=1151 piece=▁thirty\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=11 min_freq=8\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=11 size=6720 all=23801 active=1174 piece=▁dispers\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=11 size=6740 all=23787 active=1160 piece=▁searche\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=11 size=6760 all=23771 active=1144 piece=▁breastes\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=11 size=6780 all=23755 active=1128 piece=▁almightie\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=10 size=6800 all=23737 active=1110 piece=Th\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=10 min_freq=8\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=10 size=6820 all=23786 active=1232 piece=ephe\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=10 size=6840 all=23823 active=1269 piece=▁Zab\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=10 size=6860 all=23856 active=1302 piece=cking\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=10 size=6880 all=23887 active=1333 piece=▁Ezer\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=10 size=6900 all=23899 active=1345 piece=▁mils\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=10 min_freq=7\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=10 size=6920 all=23916 active=1211 piece=rouers\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=10 size=6940 all=23910 active=1205 piece=▁accou\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=10 size=6960 all=23911 active=1206 piece=▁rayse\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=10 size=6980 all=23904 active=1199 piece=▁compla\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=10 size=7000 all=23897 active=1192 piece=▁taryed\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=10 min_freq=7\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=10 size=7020 all=23894 active=1192 piece=▁calling\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=10 size=7040 all=23882 active=1180 piece=▁pearles\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=10 size=7060 all=23865 active=1163 piece=▁diminish\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=10 size=7080 all=23848 active=1146 piece=▁execrable\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=10 size=7100 all=23832 active=1130 piece=▁instrument\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=10 min_freq=7\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=10 size=7120 all=23814 active=1174 piece=▁vnrighteousnes\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=9 size=7140 all=23873 active=1233 piece=▁ta\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=9 size=7160 all=23914 active=1274 piece=sest\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=9 size=7180 all=23952 active=1312 piece=▁Tel\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=9 size=7200 all=23982 active=1342 piece=iners\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=9 min_freq=7\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=9 size=7220 all=23994 active=1209 piece=▁True\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=9 size=7240 all=23996 active=1211 piece=adezer\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=9 size=7260 all=23996 active=1211 piece=▁barre\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=9 size=7280 all=23994 active=1209 piece=▁relig\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=9 size=7300 all=23992 active=1207 piece=▁Molech\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=9 min_freq=6\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=9 size=7320 all=23982 active=1189 piece=▁holden\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=9 size=7340 all=23968 active=1175 piece=▁weapon\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=9 size=7360 all=23951 active=1158 piece=▁contend\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=9 size=7380 all=23935 active=1142 piece=▁sendeth\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=9 size=7400 all=23923 active=1130 piece=▁Maachath\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=9 min_freq=6\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=9 size=7420 all=23907 active=1178 piece=▁teachers\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=9 size=7440 all=23890 active=1161 piece=▁dishonour\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=9 size=7460 all=23870 active=1141 piece=▁birthright\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=8 size=7480 all=23860 active=1131 piece=co\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=8 size=7500 all=23929 active=1200 piece=ira\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=8 min_freq=6\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=8 size=7520 all=23970 active=1236 piece=allu\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=8 size=7540 all=24012 active=1278 piece=ouah\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=8 size=7560 all=24041 active=1307 piece=▁alt\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=8 size=7580 all=24064 active=1330 piece=isons\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=8 size=7600 all=24078 active=1344 piece=▁Meth\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=8 min_freq=6\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=8 size=7620 all=24080 active=1204 piece=▁main\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=8 size=7640 all=24104 active=1228 piece=formed\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=8 size=7660 all=24105 active=1229 piece=▁Habel\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=8 size=7680 all=24092 active=1216 piece=▁burst\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=8 size=7700 all=24084 active=1208 piece=▁robes\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=8 min_freq=6\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=8 size=7720 all=24076 active=1197 piece=▁Iether\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=8 size=7740 all=24061 active=1182 piece=▁excomm\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=8 size=7760 all=24045 active=1166 piece=▁parted\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=8 size=7780 all=24034 active=1155 piece=▁tokens\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=8 size=7800 all=24019 active=1140 piece=▁Meshech\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=8 min_freq=6\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=8 size=7820 all=24005 active=1187 piece=▁himself\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=8 size=7840 all=23990 active=1172 piece=▁stamped\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=8 size=7860 all=23973 active=1155 piece=▁almondes\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=8 size=7880 all=23957 active=1139 piece=▁reported\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=8 size=7900 all=23939 active=1121 piece=▁companies\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=8 min_freq=6\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=8 size=7920 all=23924 active=1182 piece=▁tradition\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=8 size=7940 all=23904 active=1162 piece=▁reconciled\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=8 size=7960 all=23884 active=1142 piece=▁transgressour\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=7 size=7980 all=23920 active=1178 piece=yri\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=7 size=8000 all=23961 active=1219 piece=ldad\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=7 min_freq=6\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=7 size=8020 all=23982 active=1218 piece=▁Lud\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=7 size=8040 all=24008 active=1244 piece=▁vie\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=7 size=8060 all=24041 active=1277 piece=saida\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=7 size=8080 all=24050 active=1286 piece=▁Thin\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=7 size=8100 all=24056 active=1292 piece=▁slue\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=7 min_freq=5\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=7 size=8120 all=24065 active=1212 piece=▁Exalt\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=7 size=8140 all=24051 active=1198 piece=▁calme\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=7 size=8160 all=24051 active=1198 piece=▁plowe\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=7 size=8180 all=24043 active=1190 piece=▁wiser\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=7 size=8200 all=24030 active=1177 piece=▁Stande\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=7 min_freq=5\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=7 size=8220 all=24020 active=1192 piece=▁curses\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=7 size=8240 all=24002 active=1174 piece=▁raging\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=7 size=8260 all=23989 active=1161 piece=▁vanish\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=7 size=8280 all=23974 active=1146 piece=▁boundes\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=7 size=8300 all=23964 active=1136 piece=▁inwarde\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=7 min_freq=5\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=7 size=8320 all=23950 active=1185 piece=▁shaking\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=7 size=8340 all=23931 active=1166 piece=▁Preacher\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=7 size=8360 all=23911 active=1146 piece=▁heauines\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=7 size=8380 all=23894 active=1129 piece=▁souldier\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=7 size=8400 all=23879 active=1114 piece=▁deliuerer\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=7 min_freq=5\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=7 size=8420 all=23859 active=1174 piece=▁soiourner\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=7 size=8440 all=23842 active=1157 piece=▁nakednesse\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=7 size=8460 all=23823 active=1138 piece=▁comandements\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=8480 all=23848 active=1163 piece=ita\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=8500 all=23895 active=1210 piece=▁ep\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=6 min_freq=5\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=8520 all=23935 active=1232 piece=fers\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=8540 all=23964 active=1261 piece=▁Aru\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=8560 all=23997 active=1294 piece=▁Nob\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=8580 all=24009 active=1306 piece=▁yel\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=8600 all=24027 active=1324 piece=outes\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=6 min_freq=5\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=8620 all=24032 active=1204 piece=▁Fill\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=8640 all=24030 active=1202 piece=▁bill\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=8660 all=24034 active=1206 piece=▁laie\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=8680 all=24027 active=1199 piece=▁swim\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=8700 all=24030 active=1202 piece=▁Achan\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=6 min_freq=5\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=8720 all=24020 active=1192 piece=▁Share\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=8740 all=24020 active=1192 piece=▁egges\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=8760 all=24010 active=1182 piece=▁pilgr\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=8780 all=24017 active=1189 piece=Written\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=8800 all=24016 active=1188 piece=▁Gouern\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=6 min_freq=5\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=8820 all=24005 active=1189 piece=▁careth\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=8840 all=23990 active=1174 piece=▁ioyous\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=8860 all=23982 active=1166 piece=▁remain\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=8880 all=23967 active=1151 piece=▁tossed\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=8900 all=23954 active=1138 piece=▁Goliath\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=6 min_freq=4\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=8920 all=23941 active=1185 piece=▁besomes\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=8940 all=23927 active=1171 piece=▁feruent\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=8960 all=23913 active=1157 piece=▁pottage\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=8980 all=23899 active=1143 piece=▁thrones\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=9000 all=23884 active=1128 piece=▁Herodias\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=6 min_freq=4\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=9020 all=23864 active=1175 piece=▁clusters\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=9040 all=23846 active=1157 piece=▁marueile\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=9060 all=23826 active=1137 piece=▁sufficed\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=9080 all=23808 active=1119 piece=▁broydered\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=9100 all=23792 active=1103 piece=▁preuented\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=6 min_freq=4\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=9120 all=23774 active=1172 piece=▁counseller\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=9140 all=23754 active=1152 piece=▁continuance\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=6 size=9160 all=23734 active=1132 piece=▁maliciousnes\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9180 all=23756 active=1154 piece=ara\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9200 all=23791 active=1189 piece=▁Oc\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=5 min_freq=4\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9220 all=23805 active=1203 piece=ezer\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9240 all=23832 active=1230 piece=plex\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9260 all=23849 active=1247 piece=▁Ian\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9280 all=23870 active=1268 piece=▁ash\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9300 all=23890 active=1288 piece=aired\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=5 min_freq=4\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9320 all=23905 active=1209 piece=imath\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9340 all=23926 active=1230 piece=uiour\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9360 all=23925 active=1229 piece=▁Gall\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9380 all=23927 active=1231 piece=▁Tham\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9400 all=23926 active=1230 piece=▁iest\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=5 min_freq=4\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9420 all=23938 active=1206 piece=▁trim\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9440 all=23948 active=1216 piece=▁Achor\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9460 all=23935 active=1203 piece=▁Gozan\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9480 all=23920 active=1188 piece=▁Rekem\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9500 all=23915 active=1183 piece=▁citiz\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=5 min_freq=4\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9520 all=23909 active=1188 piece=▁hauen\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9540 all=23905 active=1184 piece=▁print\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9560 all=23893 active=1172 piece=athites\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9580 all=23891 active=1170 piece=▁Dishan\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9600 all=23874 active=1153 piece=▁Rahels\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=5 min_freq=4\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9620 all=23859 active=1179 piece=▁cister\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9640 all=23851 active=1171 piece=▁fadeth\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9660 all=23837 active=1157 piece=▁madnes\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9680 all=23824 active=1144 piece=▁reuile\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9700 all=23806 active=1126 piece=plexitie\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=5 min_freq=4\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9720 all=23791 active=1175 piece=▁Mizraim\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9740 all=23773 active=1157 piece=▁ceasing\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9760 all=23756 active=1140 piece=▁fleshly\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9780 all=23738 active=1122 piece=▁painted\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9800 all=23719 active=1103 piece=▁slander\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=5 min_freq=4\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9820 all=23704 active=1170 piece=▁watches\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9840 all=23688 active=1154 piece=▁auncient\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9860 all=23672 active=1138 piece=▁fighteth\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9880 all=23655 active=1121 piece=▁passages\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9900 all=23638 active=1104 piece=▁stirreth\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=5 min_freq=4\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9920 all=23619 active=1163 piece=▁Nicodemus\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9940 all=23602 active=1146 piece=▁firebrand\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9960 all=23583 active=1127 piece=▁prouoking\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=9980 all=23569 active=1113 piece=▁beforetime\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=10000 all=23550 active=1094 piece=▁profession\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=5 min_freq=4\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=10020 all=23530 active=1158 piece=▁Girgashites\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=5 size=10040 all=23510 active=1138 piece=▁simplicitie\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10060 all=23516 active=1144 piece=ny\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10080 all=23546 active=1174 piece=que\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10100 all=23568 active=1196 piece=areb\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=4 min_freq=3\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10120 all=23593 active=1202 piece=holy\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10140 all=23611 active=1220 piece=sons\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10160 all=23624 active=1233 piece=▁Elp\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10180 all=23640 active=1249 piece=▁Wil\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10200 all=23650 active=1259 piece=▁lan\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=4 min_freq=3\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10220 all=23665 active=1196 piece=annim\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10240 all=23682 active=1213 piece=heres\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10260 all=23700 active=1231 piece=ppers\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10280 all=23718 active=1249 piece=▁Dout\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10300 all=23715 active=1246 piece=▁Slay\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=4 min_freq=3\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10320 all=23724 active=1195 piece=▁elme\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10340 all=23718 active=1189 piece=▁pipe\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10360 all=23716 active=1187 piece=abzeel\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10380 all=23733 active=1204 piece=sepher\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10400 all=23717 active=1188 piece=▁Golan\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=4 min_freq=3\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10420 all=23710 active=1179 piece=▁Patri\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10440 all=23702 active=1171 piece=▁birds\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10460 all=23698 active=1167 piece=▁enbau\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10480 all=23693 active=1162 piece=▁lowly\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10500 all=23678 active=1147 piece=▁shoot\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=4 min_freq=3\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10520 all=23668 active=1172 piece=▁wagge\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10540 all=23666 active=1170 piece=▁Ahiman\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10560 all=23654 active=1158 piece=▁Hakkoz\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10580 all=23636 active=1140 piece=▁Migdol\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10600 all=23621 active=1125 piece=▁Shobab\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=4 min_freq=3\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10620 all=23604 active=1165 piece=▁begger\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10640 all=23590 active=1151 piece=▁denyed\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10660 all=23578 active=1139 piece=▁gourde\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10680 all=23560 active=1121 piece=▁pained\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10700 all=23545 active=1106 piece=▁stumpe\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=4 min_freq=3\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10720 all=23528 active=1161 piece=aithfull\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10740 all=23514 active=1147 piece=▁Hallelu\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10760 all=23495 active=1128 piece=▁Shemida\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10780 all=23481 active=1114 piece=▁callest\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10800 all=23464 active=1097 piece=▁errours\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=4 min_freq=3\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10820 all=23445 active=1155 piece=▁iuniper\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10840 all=23427 active=1137 piece=▁pearced\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10860 all=23410 active=1120 piece=▁shekell\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10880 all=23393 active=1103 piece=▁warning\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10900 all=23383 active=1093 piece=▁Hitherto\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=4 min_freq=3\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10920 all=23364 active=1151 piece=▁anothers\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10940 all=23344 active=1131 piece=▁doctours\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10960 all=23326 active=1113 piece=▁nephewes\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=10980 all=23308 active=1095 piece=▁stumbled\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=11000 all=23290 active=1077 piece=▁Comforter\n",
      "bpe_model_trainer.cc(159) LOG(INFO) Updating active symbols. max_freq=4 min_freq=3\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=11020 all=23270 active=1145 piece=▁Withdrawe\n",
      "bpe_model_trainer.cc(268) LOG(INFO) Added: freq=4 size=11040 all=23252 active="
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[8], line 9\u001b[0m\n\u001b[1;32m      6\u001b[0m vocab_size_en \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m32000\u001b[39m  \u001b[38;5;66;03m# Set your desired vocabulary size\u001b[39;00m\n\u001b[1;32m      8\u001b[0m \u001b[38;5;66;03m# Train the English tokenizer\u001b[39;00m\n\u001b[0;32m----> 9\u001b[0m \u001b[43mspm\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mSentencePieceTrainer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m     10\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcorpus_file_en\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     11\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmodel_prefix\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmodel_prefix_en\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     12\u001b[0m \u001b[43m    \u001b[49m\u001b[43mvocab_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mvocab_size_en\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     13\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmodel_type\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mbpe\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m     14\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcharacter_coverage\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0.9995\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m  \u001b[49m\n\u001b[1;32m     15\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmax_sentence_length\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m10000\u001b[39;49m\n\u001b[1;32m     16\u001b[0m \u001b[43m)\u001b[49m\n\u001b[1;32m     18\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mEnglish tokenizer trained and saved as \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmodel_prefix_en\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.model and \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmodel_prefix_en\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.vocab\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "File \u001b[0;32m~/.local/lib/python3.10/site-packages/sentencepiece/__init__.py:1047\u001b[0m, in \u001b[0;36mSentencePieceTrainer.Train\u001b[0;34m(arg, logstream, **kwargs)\u001b[0m\n\u001b[1;32m   1044\u001b[0m \u001b[38;5;129m@staticmethod\u001b[39m\n\u001b[1;32m   1045\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mTrain\u001b[39m(arg\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, logstream\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m   1046\u001b[0m   \u001b[38;5;28;01mwith\u001b[39;00m _LogStream(ostream\u001b[38;5;241m=\u001b[39mlogstream):\n\u001b[0;32m-> 1047\u001b[0m     \u001b[43mSentencePieceTrainer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_Train\u001b[49m\u001b[43m(\u001b[49m\u001b[43marg\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43marg\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.local/lib/python3.10/site-packages/sentencepiece/__init__.py:1040\u001b[0m, in \u001b[0;36mSentencePieceTrainer._Train\u001b[0;34m(arg, **kwargs)\u001b[0m\n\u001b[1;32m   1038\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m SentencePieceTrainer\u001b[38;5;241m.\u001b[39m_TrainFromMap2(new_kwargs, sentence_iterator)\n\u001b[1;32m   1039\u001b[0m   \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1040\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mSentencePieceTrainer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_TrainFromMap\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnew_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1042\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/.local/lib/python3.10/site-packages/sentencepiece/__init__.py:985\u001b[0m, in \u001b[0;36mSentencePieceTrainer._TrainFromMap\u001b[0;34m(args)\u001b[0m\n\u001b[1;32m    983\u001b[0m \u001b[38;5;129m@staticmethod\u001b[39m\n\u001b[1;32m    984\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_TrainFromMap\u001b[39m(args):\n\u001b[0;32m--> 985\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_sentencepiece\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mSentencePieceTrainer__TrainFromMap\u001b[49m\u001b[43m(\u001b[49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import sentencepiece as spm\n",
    "\n",
    "# Define paths and parameters for English tokenizer\n",
    "corpus_file_en = 'eng-enggnv.txt'  # Path to your English corpus\n",
    "model_prefix_en = 'english_bpe'  # Prefix for the output model files\n",
    "vocab_size_en = 32000  # Set your desired vocabulary size\n",
    "\n",
    "# Train the English tokenizer\n",
    "spm.SentencePieceTrainer.train(\n",
    "    input=corpus_file_en,\n",
    "    model_prefix=model_prefix_en,\n",
    "    vocab_size=vocab_size_en,\n",
    "    model_type='bpe',\n",
    "    character_coverage=0.9995,  \n",
    "    max_sentence_length=10000\n",
    ")\n",
    "\n",
    "print(f\"English tokenizer trained and saved as {model_prefix_en}.model and {model_prefix_en}.vocab\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# preparing paraellel corpus\n",
    "def prepare_parallel_corpus(eng_file, ben_file, output_file):\n",
    "    with open(eng_file, 'r', encoding='utf-8') as eng, open(ben_file, 'r', encoding='utf-8') as ben, open(output_file, 'w', encoding='utf-8') as out:\n",
    "        eng_lines = eng.readlines()\n",
    "        ben_lines = ben.readlines()\n",
    "        \n",
    "      \n",
    "        assert len(eng_lines) == len(ben_lines), \"The English and Bengali files must have the same number of lines.\"\n",
    "        \n",
    "        for eng_line, ben_line in zip(eng_lines, ben_lines):\n",
    "            eng_line = eng_line.strip()\n",
    "            ben_line = ben_line.strip()\n",
    "            if eng_line and ben_line: \n",
    "                out.write(f\"{eng_line} ||| {ben_line}\\n\")\n",
    "\n",
    "prepare_parallel_corpus('eng-enggnv.txt', 'ben-ben2017.txt', 'parallel_corpus.txt')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  Aligning the Parallel Corpus Using FastAlign:\n",
    "import os\n",
    "\n",
    "fast_align_dir = 'fast_align/build'  \n",
    "\n",
    "os.system(f'chmod +x {fast_align_dir}/fast_align')\n",
    "os.system(f'chmod +x {fast_align_dir}/atools')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ARG=i\n",
      "ARG=d\n",
      "ARG=o\n",
      "ARG=v\n",
      "INITIAL PASS \n",
      "..............................\n",
      "expected target length = source length * 0.827891\n",
      "ITERATION 1\n",
      "..............................\n",
      "  log_e likelihood: -1.3105e+07\n",
      "  log_2 likelihood: -1.89066e+07\n",
      "     cross entropy: 29.8974\n",
      "        perplexity: 1e+09\n",
      "      posterior p0: 0.08\n",
      " posterior al-feat: -0.167949\n",
      "       size counts: 1679\n",
      "ITERATION 2\n",
      "..............................\n",
      "  log_e likelihood: -4.77458e+06\n",
      "  log_2 likelihood: -6.88827e+06\n",
      "     cross entropy: 10.8926\n",
      "        perplexity: 1901.01\n",
      "      posterior p0: 0.0820574\n",
      " posterior al-feat: -0.138536\n",
      "       size counts: 1679\n",
      "  1  model al-feat: -0.213993 (tension=4)\n",
      "  2  model al-feat: -0.17441 (tension=5.50914)\n",
      "  3  model al-feat: -0.160126 (tension=6.22663)\n",
      "  4  model al-feat: -0.152734 (tension=6.65843)\n",
      "  5  model al-feat: -0.14834 (tension=6.9424)\n",
      "  6  model al-feat: -0.145519 (tension=7.13849)\n",
      "  7  model al-feat: -0.143616 (tension=7.27816)\n",
      "  8  model al-feat: -0.142287 (tension=7.37976)\n",
      "     final tension: 7.45478\n",
      "ITERATION 3\n",
      "..............................\n",
      "  log_e likelihood: -3.6099e+06\n",
      "  log_2 likelihood: -5.20799e+06\n",
      "     cross entropy: 8.2355\n",
      "        perplexity: 301.392\n",
      "      posterior p0: 0.0537438\n",
      " posterior al-feat: -0.113005\n",
      "       size counts: 1679\n",
      "  1  model al-feat: -0.141336 (tension=7.45478)\n",
      "  2  model al-feat: -0.135006 (tension=8.0214)\n",
      "  3  model al-feat: -0.131204 (tension=8.46142)\n",
      "  4  model al-feat: -0.128898 (tension=8.82541)\n",
      "  5  model al-feat: -0.127603 (tension=9.14328)\n",
      "  6  model al-feat: -0.127104 (tension=9.43525)\n",
      "  7  model al-feat: -0.127359 (tension=9.71725)\n",
      "  8  model al-feat: -0.128502 (tension=10.0043)\n",
      "     final tension: 10.3143\n",
      "ITERATION 4\n",
      "..............................\n",
      "  log_e likelihood: -3.41827e+06\n",
      "  log_2 likelihood: -4.93152e+06\n",
      "     cross entropy: 7.79831\n",
      "        perplexity: 222.599\n",
      "      posterior p0: 0.0542917\n",
      " posterior al-feat: -0.102586\n",
      "       size counts: 1679\n",
      "  1  model al-feat: -0.13094 (tension=10.3143)\n",
      "  2  model al-feat: -0.139635 (tension=10.8814)\n",
      "  3  model al-feat: -0.163294 (tension=11.6224)\n",
      "  4  model al-feat: -0.260665 (tension=12.8365)\n",
      "  5  model al-feat: -0.512106 (tension=14)\n",
      "  6  model al-feat: -0.512106 (tension=14)\n",
      "  7  model al-feat: -0.512106 (tension=14)\n",
      "  8  model al-feat: -0.512106 (tension=14)\n",
      "     final tension: 14\n",
      "ITERATION 5 (FINAL)\n",
      "..............................\n",
      "  log_e likelihood: -3.39908e+06\n",
      "  log_2 likelihood: -4.90383e+06\n",
      "     cross entropy: 7.75453\n",
      "        perplexity: 215.947\n",
      "      posterior p0: 0\n",
      " posterior al-feat: 0\n",
      "       size counts: 1679\n",
      "ARG=i\n",
      "ARG=d\n",
      "ARG=o\n",
      "ARG=v\n",
      "ARG=r\n",
      "INITIAL PASS \n",
      "..............................\n",
      "expected target length = source length * 1.28604\n",
      "ITERATION 1\n",
      "..............................\n",
      "  log_e likelihood: -1.62018e+07\n",
      "  log_2 likelihood: -2.33743e+07\n",
      "     cross entropy: 29.8974\n",
      "        perplexity: 1e+09\n",
      "      posterior p0: 0.08\n",
      " posterior al-feat: -0.166751\n",
      "       size counts: 1679\n",
      "ITERATION 2\n",
      "..............................\n",
      "  log_e likelihood: -4.92892e+06\n",
      "  log_2 likelihood: -7.11093e+06\n",
      "     cross entropy: 9.09537\n",
      "        perplexity: 546.991\n",
      "      posterior p0: 0.0849255\n",
      " posterior al-feat: -0.134676\n",
      "       size counts: 1679\n",
      "  1  model al-feat: -0.136578 (tension=4)\n",
      "  2  model al-feat: -0.135871 (tension=4.03804)\n",
      "  3  model al-feat: -0.135429 (tension=4.06193)\n",
      "  4  model al-feat: -0.135151 (tension=4.077)\n",
      "  5  model al-feat: -0.134977 (tension=4.08651)\n",
      "  6  model al-feat: -0.134866 (tension=4.09253)\n",
      "  7  model al-feat: -0.134796 (tension=4.09634)\n",
      "  8  model al-feat: -0.134752 (tension=4.09875)\n",
      "     final tension: 4.10028\n",
      "ITERATION 3\n",
      "..............................\n",
      "  log_e likelihood: -3.83958e+06\n",
      "  log_2 likelihood: -5.53934e+06\n",
      "     cross entropy: 7.0852\n",
      "        perplexity: 135.787\n",
      "      posterior p0: 0.0594247\n",
      " posterior al-feat: -0.135339\n",
      "       size counts: 1679\n",
      "  1  model al-feat: -0.134724 (tension=4.10028)\n",
      "  2  model al-feat: -0.13495 (tension=4.08798)\n",
      "  3  model al-feat: -0.135093 (tension=4.08019)\n",
      "  4  model al-feat: -0.135183 (tension=4.07526)\n",
      "  5  model al-feat: -0.135241 (tension=4.07215)\n",
      "  6  model al-feat: -0.135277 (tension=4.07018)\n",
      "  7  model al-feat: -0.1353 (tension=4.06893)\n",
      "  8  model al-feat: -0.135314 (tension=4.06815)\n",
      "     final tension: 4.06765\n",
      "ITERATION 4\n",
      "..............................\n",
      "  log_e likelihood: -3.64197e+06\n",
      "  log_2 likelihood: -5.25425e+06\n",
      "     cross entropy: 6.72056\n",
      "        perplexity: 105.46\n",
      "      posterior p0: 0.0623612\n",
      " posterior al-feat: -0.135531\n",
      "       size counts: 1679\n",
      "  1  model al-feat: -0.135323 (tension=4.06765)\n",
      "  2  model al-feat: -0.1354 (tension=4.06351)\n",
      "  3  model al-feat: -0.135448 (tension=4.06089)\n",
      "  4  model al-feat: -0.135479 (tension=4.05923)\n",
      "  5  model al-feat: -0.135498 (tension=4.05819)\n",
      "  6  model al-feat: -0.13551 (tension=4.05753)\n",
      "  7  model al-feat: -0.135518 (tension=4.05712)\n",
      "  8  model al-feat: -0.135523 (tension=4.05686)\n",
      "     final tension: 4.05669\n",
      "ITERATION 5 (FINAL)\n",
      "..............................\n",
      "  log_e likelihood: -3.58751e+06\n",
      "  log_2 likelihood: -5.17569e+06\n",
      "     cross entropy: 6.62007\n",
      "        perplexity: 98.3647\n",
      "      posterior p0: 0\n",
      " posterior al-feat: 0\n",
      "       size counts: 1679\n"
     ]
    }
   ],
   "source": [
    "#  Aligning the Parallel Corpus Using FastAlign:\n",
    "!{fast_align_dir}/fast_align -i parallel_corpus.txt -d -o -v > forward.align\n",
    "\n",
    "!{fast_align_dir}/fast_align -i parallel_corpus.txt -d -o -v -r > reverse.align\n",
    "\n",
    "!{fast_align_dir}/atools -i forward.align -j reverse.align -c grow-diag-final-and > bidirectional.align\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Forward Alignments \n",
      "['2-0 3-1 4-2 5-3 6-4 4-5 9-6\\n', '2-0 2-1 4-2 6-3 5-4 10-5 8-6 9-7 10-8 11-9 10-10 14-11 16-12 16-13 22-14 20-15 22-16 22-17\\n', '0-0 1-1 2-2 6-3 6-4 7-5 10-6 10-7\\n', '0-0 1-1 4-2 8-3 2-4 9-5 10-6 13-7 12-8 13-9 11-10 16-11\\n', '0-0 1-1 4-2 2-3 5-4 6-5 8-6 10-7 11-8 10-9 14-10 15-11 17-12 17-13 20-14 21-15 21-16\\n']\n",
      "\n",
      "Reverse Alignments \n",
      "['0-0 1-0 2-0 3-1 4-5 5-4 6-2 7-3 8-5 9-4\\n', '0-11 1-0 2-0 3-5 4-2 5-4 6-3 7-4 8-6 9-7 10-5 11-9 12-9 13-14 14-6 15-13 16-13 17-12 18-12 19-16 20-15 21-15 22-14\\n', '0-0 1-1 2-2 3-4 4-3 5-4 6-3 7-5 8-6 9-7 10-6\\n', '0-0 1-1 2-4 3-2 4-2 5-4 6-4 7-4 8-3 9-5 10-6 11-10 12-8 13-7 14-10 15-10 16-9\\n', '0-0 1-1 2-3 3-2 4-4 5-4 6-5 7-5 8-6 10-7 11-8 12-10 13-9 14-10 15-11 16-12 17-12 18-16 19-14 20-14 21-16\\n']\n",
      "\n",
      "Bidirectional Alignments:\n",
      "['0-0 1-0 2-0 3-1 4-2 4-5 5-3 5-4 6-2 7-3 9-6\\n', '1-0 2-0 2-1 4-2 5-4 6-3 7-4 8-6 9-7 10-5 10-8 10-10 11-9 12-9 14-11 15-13 16-12 16-13 17-12 18-12 19-16 20-15 21-15 22-14\\n', '0-0 1-1 2-2 3-4 4-3 5-4 6-3 7-5 8-6 9-7 10-6\\n', '0-0 1-1 2-4 3-2 4-2 5-4 6-4 7-4 8-3 9-5 10-6 11-10 12-8 13-7 13-9 14-10 15-10 16-9 16-11\\n', '0-0 1-1 2-3 3-2 4-2 5-4 6-5 7-5 8-6 10-7 10-9 11-8 12-10 13-9 14-10 15-11 16-12 17-12 17-13 19-14 20-14 21-15 21-16\\n']\n"
     ]
    }
   ],
   "source": [
    "def read_alignment_file(filepath):\n",
    "    with open(filepath, 'r', encoding='utf-8') as file:\n",
    "        alignments = file.readlines()\n",
    "    return alignments\n",
    "\n",
    "forward_alignments = read_alignment_file('forward.align')\n",
    "reverse_alignments = read_alignment_file('reverse.align')\n",
    "bidirectional_alignments = read_alignment_file('bidirectional.align')\n",
    "\n",
    "\n",
    "print(\"Forward Alignments \")\n",
    "print(forward_alignments[:5])\n",
    "\n",
    "print(\"\\nReverse Alignments \")\n",
    "print(reverse_alignments[:5])\n",
    "\n",
    "print(\"\\nBidirectional Alignments:\")\n",
    "print(bidirectional_alignments[:5])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "import sentencepiece as spm\n",
    "\n",
    "\n",
    "def read_alignment_file(filepath):\n",
    "    with open(filepath, 'r', encoding='utf-8') as file:\n",
    "        alignments = file.readlines()\n",
    "    return alignments\n",
    "\n",
    "forward_alignments = read_alignment_file('forward.align')\n",
    "reverse_alignments = read_alignment_file('reverse.align')\n",
    "bidirectional_alignments = read_alignment_file('bidirectional.align')\n",
    "\n",
    "def create_token_mapping(alignments):\n",
    "    token_map = defaultdict(list)\n",
    "    for line in alignments:\n",
    "        parts = line.strip().split()\n",
    "        for part in parts:\n",
    "            en_idx, bn_idx = part.split('-')\n",
    "            token_map[int(en_idx)].append(int(bn_idx))\n",
    "    return token_map\n",
    "\n",
    "\n",
    "token_map = create_token_mapping(bidirectional_alignments)\n",
    "\n",
    "print(token_map)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#failing to create smooth token mapping"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}