poetry_generation/Simple_lstm.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "cfdc573f",
   "metadata": {},
   "source": [
    "Adapted from: https://gist.github.com/maxim5/c35ef2238ae708ccb0e55624e9e0252b#file-pretrained_word2vec_lstm_gen-py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "bca53c9e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "import itertools\n",
    "import json\n",
    "from pathlib import Path\n",
    "import pickle\n",
    "import random\n",
    "import string\n",
    "\n",
    "import gensim\n",
    "import json\n",
    "import nltk\n",
    "import numpy as np\n",
    "import zipfile\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint, CSVLogger, LambdaCallback, TerminateOnNaN\n",
    "from tensorflow.keras.layers import Dense, Activation, LSTM, Embedding, Masking\n",
    "from tensorflow.keras.models import Sequential, load_model\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer, tokenizer_from_json\n",
    "\n",
    "SEED = 420\n",
    "# sets random, np.random and tf.random seed\n",
    "tf.keras.utils.set_random_seed(\n",
    "    SEED\n",
    ")\n",
    "\n",
    "MAX_LEN=30"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "aef6c436",
   "metadata": {},
   "outputs": [],
   "source": [
    "punctuation = string.punctuation + \"«»—...\"\n",
    "\n",
    "def tokenize_line(line, sos, eos, lower):\n",
    "    if lower:\n",
    "        tokens = [t.lower() for t in nltk.tokenize.word_tokenize(line, language='norwegian') if t not in punctuation]\n",
    "    else:\n",
    "        tokens = [t for t in nltk.tokenize.word_tokenize(line, language='norwegian') if t not in punctuation]\n",
    "\n",
    "    if sos:\n",
    "        tokens = [sos] + tokens \n",
    "    if eos:\n",
    "        tokens = tokens + [eos]\n",
    "    return tokens\n",
    "\n",
    "def get_sentences(path, pad = \"<pad>\", sos=\"<s>\", eos=\"</s>\", lower=True):\n",
    "    p = Path(path)\n",
    "    sentences = []\n",
    "    \n",
    "    for e in p.iterdir():\n",
    "        content = e.read_text()\n",
    "        stanzas = content.split(\"\\n\\n\")\n",
    "        for stanza in stanzas:\n",
    "            for line in stanza.split(\"\\n\"):\n",
    "                if line:\n",
    "                    sentences.append(tokenize_line(line, sos, eos, lower))\n",
    "    if pad:\n",
    "        sentences = [[pad]] + sentences\n",
    "    return sentences\n",
    "\n",
    "def tokenize_stanza(stanza, sos, eos, newline, lower):\n",
    "    lines = stanza.split(\"\\n\")\n",
    "    stanza = [sos]\n",
    "    if lower:\n",
    "        for line in lines:\n",
    "            stanza += [t.lower() for t in nltk.tokenize.word_tokenize(line, language='norwegian') if t not in punctuation] + [newline]\n",
    "    else:\n",
    "        for line in lines:\n",
    "            stanza += [t for t in nltk.tokenize.word_tokenize(line, language='norwegian') if t not in punctuation]+[newline]\n",
    "    stanza += [eos]\n",
    "    return stanza\n",
    "\n",
    "def get_stanzas(path, pad = \"<pad>\", sos=\"<s>\", eos=\"</s>\", newline=\"<\\n>\", lower=True):\n",
    "    p = Path(path)\n",
    "    stanzas = []\n",
    "    \n",
    "    for e in p.iterdir():\n",
    "        content = e.read_text()\n",
    "        stanzas_ = content.split(\"\\n\\n\")\n",
    "        for stanza in stanzas_:\n",
    "            stanzas.append(tokenize_stanza(stanza, sos, eos, newline, lower))\n",
    "    if pad:\n",
    "        stanzas = [[pad]] + stanzas\n",
    "    return stanzas\n",
    "\n",
    "def train_word2vec(sentences):\n",
    "    word_model = gensim.models.Word2Vec(sentences, vector_size=100, min_count=1, window=5, epochs=100, sorted_vocab=0)\n",
    "    return word_model\n",
    "\n",
    "def get_train_data(sentences, wv, pad_idx = 0):\n",
    "    index_sentences = [[wv.key_to_index[word] for word in sentence] for sentence in sentences]\n",
    "    x = []\n",
    "    y = []\n",
    "    \n",
    "    for sentence in index_sentences:\n",
    "        for i in range(1, len(sentence)):\n",
    "            x.append(sentence[:i])\n",
    "            y.append(sentence[i])\n",
    "    \n",
    "    x = pad_sequences(x, padding=\"post\", value=pad_idx) \n",
    "    y = np.array(y)\n",
    "    return x, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "a5f196cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def sample(preds, temperature=0):\n",
    "    if temperature == 0:\n",
    "        return np.argmax(preds)\n",
    "    \n",
    "    preds = np.asarray(preds).astype('float64')\n",
    "    preds = np.log(preds) / temperature\n",
    "    exp_preds = np.exp(preds)\n",
    "    preds = exp_preds / np.sum(exp_preds)\n",
    "    probas = np.random.multinomial(1, preds, 1)\n",
    "    return np.argmax(probas)\n",
    "\n",
    "\n",
    "def generate_next(text, model, wv, num_generated=10, temperature=0.1, eos=None, newline=None, reverse=False):\n",
    "    word_idxs = [wv.key_to_index[word.lower()] for word in text.split()]\n",
    "    \n",
    "    for i in range(num_generated):\n",
    "        a = np.array(word_idxs)\n",
    "        b = np.reshape(a, (1, *a.shape))\n",
    "\n",
    "        prediction = model.predict(x=b)\n",
    "        idx = sample(prediction[-1], temperature)\n",
    "        word_idxs.append(idx)\n",
    "        \n",
    "        if eos:\n",
    "            if wv.index_to_key[idx] == eos:\n",
    "                break\n",
    "        if newline:\n",
    "            if wv.index_to_key[idx] == newline:\n",
    "                break\n",
    "    if reverse:\n",
    "        word_idxs.reverse()\n",
    "    return [wv.index_to_key[idx] for idx in word_idxs]    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "b7e62b72",
   "metadata": {},
   "outputs": [],
   "source": [
    "def stanza_to_string(stanza, sos, eos, newline):\n",
    "    stanza = \" \".join(stanza)\n",
    "    stanza = stanza.replace(newline, \"\\n\")\n",
    "    if sos:\n",
    "        stanza = stanza.replace(sos, \"\")\n",
    "    if eos:\n",
    "        stanza = stanza.replace(eos, \"\")\n",
    "    return stanza"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "31a81b42",
   "metadata": {},
   "outputs": [],
   "source": [
    "def lines_to_string(lines, sos, eos):\n",
    "    stanza = \"\\n\".join([\" \".join(line) for line in lines])\n",
    "    if sos:\n",
    "        stanza = stanza.replace(sos, \"\")\n",
    "    if eos:\n",
    "        stanza = stanza.replace(eos, \"\")\n",
    "    return stanza"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b0dd5d3e",
   "metadata": {},
   "source": [
    "# Reverse stanza level LSTM for rhyming poetry model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "268396ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "pad = \"<pad>\"\n",
    "sos = \"<s>\"\n",
    "eos = \"</s>\"\n",
    "newline = \"<n>\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "75eb9648",
   "metadata": {},
   "outputs": [],
   "source": [
    "stanzas = get_stanzas(\"../../norwegian_rhyme_scheme_corpus/poems/bokmål/\", pad, sos, eos, newline)\n",
    "for s in stanzas:\n",
    "    s.reverse()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "c3c24d09",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocab size: 17877\n",
      "Embedding_size: 100\n"
     ]
    }
   ],
   "source": [
    "model_name = \"poetry_gen_reverse_stanza\"\n",
    "\n",
    "# # Uncomment to train\n",
    "# word_model = train_word2vec(stanzas)\n",
    "# word_model.save(f\"models/word2vec_{model_name}.model\")\n",
    "\n",
    "word_model = gensim.models.Word2Vec.load(f\"models/word2vec_{model_name}.model\")\n",
    "\n",
    "pretrained_weights = word_model.wv.vectors\n",
    "vocab_size, embedding_size = pretrained_weights.shape\n",
    "print(f\"Vocab size: {vocab_size}\\nEmbedding_size: {embedding_size}\")\n",
    "pad_idx = word_model.wv.key_to_index[pad]\n",
    "assert pad_idx == 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "af45cfc4",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((185896, 119), (185896,))"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X, y = get_train_data(stanzas, word_model.wv, pad_idx)\n",
    "X.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "348b42d8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19238"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "8a78abf0",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model_name = \"poetry_gen_reverse_stanza\"\n",
    "\n",
    "# # uncomment to train model\n",
    "# print('\\nTraining LSTM...')\n",
    "# model = Sequential([\n",
    "#         Embedding(input_dim=vocab_size, output_dim=embedding_size, weights=[pretrained_weights], mask_zero=True),\n",
    "#         LSTM(units=embedding_size),\n",
    "#         Dense(units=vocab_size),\n",
    "#         Activation('softmax')\n",
    "#         ])\n",
    "\n",
    "\n",
    "# model.compile(optimizer='adam', loss='sparse_categorical_crossentropy')\n",
    "\n",
    "# model_checkpoint = tf.keras.callbacks.ModelCheckpoint(f\"models/{model_name}.hdf5\", monitor=\"val_loss\")\n",
    "# terminate_on_nan = tf.keras.callbacks.TerminateOnNaN()\n",
    "# csv_logger = tf.keras.callbacks.CSVLogger(f\"logs/training_{model_name}.log\")\n",
    "# early_stop = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=5)\n",
    "\n",
    "# history = model.fit(X, y,\n",
    "#                     batch_size=256,\n",
    "#                     epochs=100,\n",
    "#                     callbacks=[model_checkpoint, terminate_on_nan, csv_logger, early_stop])\n",
    "\n",
    "\n",
    "model = load_model(f\"models/{model_name}.hdf5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "cd630077",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "prompt: </s> <n> gjort du\n",
      " fra gaten her den ånd \n",
      " vil de dem i vanens natt \n",
      " har tro det skarpe fakler i brann \n",
      " og elisa ham kalte ut \n",
      " og roper han fryktelig lønn så trygg han taler \n",
      " spar dog han alene med hans makt betvinger \n",
      " og han hører roper han tilkjenne guds svar \n",
      " enn er det å men husk det ei kan trenge \n",
      " nå er det tid alt som har du gjort \n",
      " \n",
      "---\n",
      "prompt: </s> <n> blitt hadde\n",
      " da han etter kongen fyller \n",
      " seg frem den trygge hvile på \n",
      " leste så han på sitt hjerte når \n",
      " han kom på at gud må folket se \n",
      " ei barnlige sjel med 1vs derinn \n",
      " og tidlig forvirrede konger hadde blitt \n",
      " \n",
      "---\n",
      "prompt: </s>\n",
      " så ble jeg så om minne om far og om du er født ved kveld \n",
      " så du skrova med skjell og tidlig på \n",
      " før tanken stadig reiser det opp av sol i nord \n",
      " fra rom velsignede sted på et sted \n",
      " der ikke kom ikke en purk i land \n",
      " \n",
      " \n",
      "---\n",
      "prompt: </s> <n> du\n",
      " molde molde \n",
      " stammes ånd i det mørke fjell \n",
      " bestandig uten gull av jord \n",
      " men jeg har intet så han er det godt \n",
      " kan ikke slett ikke nå som du \n",
      " \n",
      "---\n"
     ]
    }
   ],
   "source": [
    "sample_sents = [\n",
    "    \"</s> <n> gjort du\",\n",
    "    \"</s> <n> blitt hadde\",\n",
    "    \"</s>\",\n",
    "    \"</s> <n> du\",\n",
    "]\n",
    "\n",
    "for prompt in sample_sents:\n",
    "    print(f\"prompt: {prompt}\")\n",
    "    gen = generate_next(prompt, model, word_model.wv, num_generated=100, temperature=0, eos=sos, reverse=True)\n",
    "    print(stanza_to_string(gen, sos, eos, newline))\n",
    "    print(\"---\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "09fa3c2a",
   "metadata": {},
   "source": [
    "## Baseline generation: no rhyme"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "35a9498b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_4531/2015149793.py:6: RuntimeWarning: divide by zero encountered in log\n",
      "  preds = np.log(preds) / temperature\n"
     ]
    }
   ],
   "source": [
    "# s = \"\"\n",
    "# for i in range(40):\n",
    "#     gen = generate_next(eos, model, word_model.wv, num_generated=100, temperature=0.5, eos=sos, reverse=True)\n",
    "#     s += stanza_to_string(gen, sos, eos, newline) + \"\\n\\n\"\n",
    "\n",
    "# with open(f\"baseline_poetry_no_rhyme_stanza.txt\", \"w+\") as f:\n",
    "#     f.write(s)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5bb14401",
   "metadata": {},
   "source": [
    "## Plug rhyme model into generate fuction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "1cc263f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_from_rhyme_scheme(scheme, model, wv, rhyme_model, temperature, sos, eos):\n",
    "    #step 1: pick out start words\n",
    "    unique_symbols = list(set(scheme))\n",
    "    start_words = {}\n",
    "    for symbol in unique_symbols:\n",
    "        seq = generate_next(eos, model, wv, num_generated=3, temperature=temperature)\n",
    "        for w in seq:\n",
    "            if w not in (eos, sos, newline):\n",
    "                start_words[symbol] = [w]\n",
    "                break\n",
    "\n",
    "    # step 2: pick out rhyming words to the start words:\n",
    "    flat_start_words = [start_words[sym][0] for sym in unique_symbols]\n",
    "    symbol_buckets = {sym: bucket for sym, bucket in zip(unique_symbols, words_to_buckets(flat_start_words))}\n",
    "    c = Counter(scheme)\n",
    "    for sym, count in c.items():\n",
    "        if count > 1:\n",
    "            bucket = symbol_buckets[sym]\n",
    "            for i in range(count-1):\n",
    "                start_words[sym].append(random.choice(bucket))\n",
    "    # step 3: put in right order\n",
    "    line_ending_words = []\n",
    "    for symbol in scheme:\n",
    "        line_ending_words.append(start_words[symbol].pop())\n",
    "    \n",
    "    # step 4: Generate!\n",
    "    line_ending_words.reverse() #stanza is generated in reverse\n",
    "    stanza = \" \".join([sos, newline])\n",
    "    for i, w in enumerate(line_ending_words):\n",
    "        s = stanza + \" \" + w\n",
    "        line = generate_next(s, model, word_model.wv, num_generated=20, eos=sos, newline=newline)\n",
    "        if line[-1] == sos:\n",
    "            line[-1] = newline\n",
    "        if i == len(scheme)-1:\n",
    "            line.reverse()\n",
    "            stanza = line\n",
    "        else:\n",
    "            stanza = \" \".join(line)\n",
    "            \n",
    "    return stanza_to_string(stanza, sos, eos, newline)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "721c7f76",
   "metadata": {},
   "source": [
    "# Load rhyme model (norsc dense >9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "6bfc46fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "rhyme_model_name = \"rhyme_gen_norsc_big_9_buckets\"\n",
    "rhyme_model = tf.keras.models.load_model(f\"../rhyme_modelling/models/{rhyme_model_name}.hdf5\")\n",
    "\n",
    "buckets_name = rhyme_model_name\n",
    "\n",
    "with open(f'../rhyme_modelling/pickles/{buckets_name}.pickle','rb') as f:\n",
    "    buckets = pickle.load(f)\n",
    "\n",
    "with open(f\"../rhyme_modelling/good+manual_char_tokenizer_config.json\") as f:\n",
    "    tokenizer_config = f.read()\n",
    "\n",
    "rhyme_char_tokenizer = tf.keras.preprocessing.text.tokenizer_from_json(tokenizer_config)\n",
    "\n",
    "def words_to_buckets(words):\n",
    "    x = rhyme_char_tokenizer.texts_to_sequences(words)    \n",
    "    x = tf.keras.preprocessing.sequence.pad_sequences(x, maxlen=60, padding=\"post\", value=0) \n",
    "    preds = rhyme_model.predict(x)\n",
    "    p = np.argmax(preds, axis=1)\n",
    "    return [buckets[i] for i in p]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "1f24babe",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "står\n",
      "['sjelesår', 'forslår', 'slavekår', 'forgår', 'trengselskår']\n",
      "\n",
      "vind\n",
      "['søndenvind', 'ditinn', 'marmortrinn', 'hjernespinn', 'stinn']\n",
      "\n",
      "spinn\n",
      "['søndenvind', 'ditinn', 'marmortrinn', 'hjernespinn', 'stinn']\n",
      "\n",
      "spe\n",
      "['avsted', 'geled', 'skje', 'galgetre', 'tre']\n",
      "\n",
      "tent\n",
      "['procent', 'forbrent', 'sendt', 'tent', 'moment']\n",
      "\n",
      "støttepunkt\n",
      "['tukt', 'flukt', 'frukt', 'produkt', 'fukt']\n",
      "\n"
     ]
    }
   ],
   "source": [
    "wo = [\"står\", \"vind\", \"spinn\", \"spe\", \"tent\", \"støttepunkt\"]\n",
    "\n",
    "for w, b in zip(wo, words_to_buckets(wo)):\n",
    "    print(w)\n",
    "    print(b[:5])\n",
    "    print(\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "440c0a3d",
   "metadata": {},
   "source": [
    "# Generate!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "0344e2c9",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_4531/2015149793.py:6: RuntimeWarning: divide by zero encountered in log\n",
      "  preds = np.log(preds) / temperature\n"
     ]
    }
   ],
   "source": [
    "# top_rhyme_schemes = [\"ABAB\", \"ABCB\", \"AABB\",\"AABCCB\", \"ABBA\", \"AABBCC\", \"AAA\", \"ABAAB\", \"AABCBC\",\"ABABCC\"]\n",
    "\n",
    "# s = \"\"\n",
    "# for scheme in top_rhyme_schemes:\n",
    "#     for i in range(4):\n",
    "#         s += scheme + gen_from_rhyme_scheme(scheme, model, word_model.wv, rhyme_model, temperature=0.5, sos=sos, eos=eos) + \"\\n\"\n",
    "    \n",
    "# with open(f\"generated_rhyming_poetry_{rhyme_model_name}_stanza.txt\", \"w+\") as f:\n",
    "#     f.write(s)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "37927b50",
   "metadata": {},
   "source": [
    "# Reverse line level generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "c2ddf379",
   "metadata": {},
   "outputs": [],
   "source": [
    "pad = \"<pad>\"\n",
    "sos = \"<s>\"\n",
    "eos=\"</s>\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "52d570b9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['</s>', 'norge', 'fra', 'svar', '<s>']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences = get_sentences(\"../../norwegian_rhyme_scheme_corpus/poems/bokmål/\", pad, sos, eos)\n",
    "\n",
    "for s in sentences:\n",
    "    s.reverse()\n",
    "    \n",
    "sentences[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "e4dcdd7e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocab size: 17876\n",
      "Embedding_size: 100\n"
     ]
    }
   ],
   "source": [
    "model_name = \"poetry_gen_reverse_line\"\n",
    "\n",
    "# # Uncomment to train\n",
    "# word_model = train_word2vec(sentences)\n",
    "# word_model.save(f\"models/word2vec_{model_name}.model\")\n",
    "\n",
    "word_model = gensim.models.Word2Vec.load(f\"models/word2vec_{model_name}.model\")\n",
    "\n",
    "pretrained_weights = word_model.wv.vectors\n",
    "vocab_size, embedding_size = pretrained_weights.shape\n",
    "print(f\"Vocab size: {vocab_size}\\nEmbedding_size: {embedding_size}\")\n",
    "pad_idx = word_model.wv.key_to_index[pad]\n",
    "assert pad_idx == 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "f3882a0c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((180231, 26), (180231,))"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X, y = get_train_data(sentences, word_model.wv, pad_idx)\n",
    "X.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "17124734",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'poetry_gen_reverse_line'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "f6d64ddd",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# # uncomment to train model\n",
    "# print('\\nTraining LSTM...')\n",
    "# model = Sequential([\n",
    "#         Embedding(input_dim=vocab_size, output_dim=embedding_size, weights=[pretrained_weights], mask_zero=True),\n",
    "#         LSTM(units=embedding_size),\n",
    "#         Dense(units=vocab_size),\n",
    "#         Activation('softmax')\n",
    "#         ])\n",
    "\n",
    "\n",
    "# model.compile(optimizer='adam', loss='sparse_categorical_crossentropy')\n",
    "\n",
    "# model_checkpoint = tf.keras.callbacks.ModelCheckpoint(f\"models/{model_name}.hdf5\",monitor=\"val_loss\")\n",
    "# terminate_on_nan = tf.keras.callbacks.TerminateOnNaN()\n",
    "# csv_logger = tf.keras.callbacks.CSVLogger(f'logs/training_{model_name}.log')\n",
    "# early_stop = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=5)\n",
    "\n",
    "# history = model.fit(X, y,\n",
    "#                     batch_size=256,\n",
    "#                     epochs=100,\n",
    "#                     callbacks=[model_checkpoint, terminate_on_nan, csv_logger, early_stop])\n",
    "\n",
    "model = load_model(f\"models/{model_name}.hdf5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "8234683e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def line_gen_from_rhyme_scheme(scheme, model, wv, rhyme_model, temperature, sos, eos):\n",
    "    #step 1: pick out start words\n",
    "    unique_symbols = list(set(scheme))\n",
    "    start_words = {}\n",
    "    for symbol in unique_symbols:\n",
    "        seq = generate_next(eos, model, wv, num_generated=3, temperature=temperature)\n",
    "        for w in seq:\n",
    "            if w not in (eos, sos):\n",
    "                start_words[symbol] = [w]\n",
    "                break\n",
    "\n",
    "    # step 2: pick out rhyming words to the start words:\n",
    "    flat_start_words = [start_words[sym][0] for sym in unique_symbols]\n",
    "    symbol_buckets = {sym: bucket for sym, bucket in zip(unique_symbols, words_to_buckets(flat_start_words))}\n",
    "    c = Counter(scheme)\n",
    "    for sym, count in c.items():\n",
    "        if count > 1:\n",
    "            bucket = symbol_buckets[sym]\n",
    "            for i in range(count-1):\n",
    "                start_words[sym].append(random.choice(bucket))\n",
    "    # step 3: put in right order\n",
    "    line_ending_words = []\n",
    "    for symbol in scheme:\n",
    "        line_ending_words.append(start_words[symbol].pop())\n",
    "    \n",
    "    # step 4: Generate!\n",
    "    lines = [generate_next(f\"{eos} {w}\", model, word_model.wv, num_generated=20, eos=sos, reverse=True) for w in line_ending_words]\n",
    "    return lines_to_string(lines, sos, eos)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f48c92fc",
   "metadata": {},
   "source": [
    "## Load rhyme model (only norsc >9 buckets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "c13bc55d",
   "metadata": {},
   "outputs": [],
   "source": [
    "rhyme_model_name = \"rhyme_gen_norsc_big_9_buckets\"\n",
    "rhyme_model = tf.keras.models.load_model(f\"../rhyme_modelling/models/{rhyme_model_name}.hdf5\")\n",
    "\n",
    "buckets_name = rhyme_model_name\n",
    "\n",
    "with open(f'../rhyme_modelling/pickles/{buckets_name}.pickle','rb') as f:\n",
    "    buckets = pickle.load(f)\n",
    "\n",
    "# with open(f\"../rhyme_modelling/merged_big_4_char_tokenizer_config.json\") as f:\n",
    "with open(f\"../rhyme_modelling/good+manual_char_tokenizer_config.json\") as f:\n",
    "    tokenizer_config = f.read()\n",
    "\n",
    "rhyme_char_tokenizer = tf.keras.preprocessing.text.tokenizer_from_json(tokenizer_config)\n",
    "\n",
    "def words_to_buckets(words):\n",
    "    x = rhyme_char_tokenizer.texts_to_sequences(words)    \n",
    "    x = tf.keras.preprocessing.sequence.pad_sequences(x, maxlen=60, padding=\"post\", value=0) \n",
    "    preds = rhyme_model.predict(x)\n",
    "    p = np.argmax(preds, axis=1)\n",
    "    return [buckets[i] for i in p]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "57802c6e",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "står\n",
      "['sjelesår', 'forslår', 'slavekår', 'forgår', 'trengselskår']\n",
      "\n",
      "vind\n",
      "['søndenvind', 'ditinn', 'marmortrinn', 'hjernespinn', 'stinn']\n",
      "\n",
      "spinn\n",
      "['søndenvind', 'ditinn', 'marmortrinn', 'hjernespinn', 'stinn']\n",
      "\n",
      "spe\n",
      "['avsted', 'geled', 'skje', 'galgetre', 'tre']\n",
      "\n",
      "tent\n",
      "['procent', 'forbrent', 'sendt', 'tent', 'moment']\n",
      "\n",
      "støttepunkt\n",
      "['tukt', 'flukt', 'frukt', 'produkt', 'fukt']\n",
      "\n"
     ]
    }
   ],
   "source": [
    "wo = [\"står\", \"vind\", \"spinn\", \"spe\", \"tent\", \"støttepunkt\"]\n",
    "\n",
    "for w, b in zip(wo, words_to_buckets(wo)):\n",
    "    print(w)\n",
    "    print(b[:5])\n",
    "    print(\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "86371110",
   "metadata": {},
   "source": [
    "# Generate!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "a251fba6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_4531/2015149793.py:6: RuntimeWarning: divide by zero encountered in log\n",
      "  preds = np.log(preds) / temperature\n"
     ]
    }
   ],
   "source": [
    "# top_rhyme_schemes = [\"ABAB\", \"ABCB\", \"AABB\",\"AABCCB\", \"ABBA\", \"AABBCC\", \"AAA\", \"ABAAB\", \"AABCBC\",\"ABABCC\"]\n",
    "\n",
    "# s = \"\"\n",
    "# for scheme in top_rhyme_schemes:\n",
    "#     for i in range(4):\n",
    "#         s += scheme + \"\\n\" + line_gen_from_rhyme_scheme(scheme, model, word_model.wv, rhyme_model, temperature=0.5, sos=sos, eos=eos) + \"\\n\\n\"\n",
    "        \n",
    "# with open(f\"generated_rhyming_poetry_{rhyme_model_name}_line.txt\", \"w+\") as f:\n",
    "#     f.write(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9df733e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c71662c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06b19c67",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c26e0239",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "57a3fb52",
   "metadata": {},
   "source": [
    "# Sanity Line level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ae1ca62c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['<pad>'],\n",
       " ['<s>', 'svar', 'fra', 'norge', '</s>'],\n",
       " ['<s>', 'har', 'du', 'hørt', 'hva', 'svensken', 'sier', '</s>'],\n",
       " ['<s>', 'unge', 'norske', 'mann', '</s>'],\n",
       " ['<s>', 'har', 'du', 'seet', 'hva', 'som', 'stiger', '</s>']]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pad = \"<pad>\"\n",
    "sos = \"<s>\"\n",
    "eos=\"</s>\"\n",
    "\n",
    "sentences = get_sentences(\"../../norwegian_rhyme_scheme_corpus/poems/bokmål/\", pad, sos, eos)\n",
    "sentences = sentences[:5]\n",
    "sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "605d0915",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocab size: 18\n",
      "Embedding size: 100\n"
     ]
    }
   ],
   "source": [
    "word_model = train_word2vec(sentences)\n",
    "pretrained_weights = word_model.wv.vectors\n",
    "vocab_size, embedding_size = pretrained_weights.shape\n",
    "print(f\"Vocab size: {vocab_size}\\nEmbedding size: {embedding_size}\")\n",
    "\n",
    "pad_idx = word_model.wv.key_to_index[pad]\n",
    "assert pad_idx == 0\n",
    "\n",
    "X, y = get_train_data(sentences, word_model.wv, pad_idx)\n",
    "X.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "396ce982",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-05-01 22:33:07.398750: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory\n",
      "2022-05-01 22:33:07.398794: W tensorflow/stream_executor/cuda/cuda_driver.cc:269] failed call to cuInit: UNKNOWN ERROR (303)\n",
      "2022-05-01 22:33:07.398818: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (tita-laptop): /proc/driver/nvidia/version does not exist\n",
      "2022-05-01 22:33:07.399176: I tensorflow/core/platform/cpu_feature_guard.cc:151] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "model_name = \"sanity\"\n",
    "\n",
    "# # uncomment to train model\n",
    "# print('\\nTraining LSTM...')\n",
    "# model = Sequential([\n",
    "#         Embedding(input_dim=vocab_size, output_dim=embedding_size, weights=[pretrained_weights], mask_zero=True),\n",
    "#         LSTM(units=embedding_size),\n",
    "#         Dense(units=vocab_size),\n",
    "#         Activation('softmax')\n",
    "#         ])\n",
    "\n",
    "\n",
    "# model.compile(optimizer='adam', loss='sparse_categorical_crossentropy')\n",
    "\n",
    "# model_checkpoint = ModelCheckpoint(f\"models/{model_name}.hdf5\",monitor=\"val_loss\")\n",
    "# terminate_on_nan = TerminateOnNaN()\n",
    "# csv_logger = CSVLogger(f'training_{model_name}.log')\n",
    "# history = model.fit(X, y,\n",
    "#           batch_size=1,\n",
    "#           epochs=20,\n",
    "#           callbacks=[model_checkpoint, terminate_on_nan])\n",
    "\n",
    "model = load_model(f\"models/{model_name}.hdf5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ecf61a10",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---\n",
      "train sent:\n",
      " <s> svar fra norge </s>\n",
      "prompt: ['<s>', 'svar', 'fra']\n",
      "['<s>', 'svar', 'fra', 'norge', '</s>']\n",
      "---\n",
      "---\n",
      "train sent:\n",
      " <s> har du hørt hva svensken sier </s>\n",
      "prompt: ['<s>', 'har', 'du']\n",
      "['<s>', 'har', 'du', 'hørt', 'hva', 'som', 'stiger', '</s>']\n",
      "---\n",
      "---\n",
      "train sent:\n",
      " <s> unge norske mann </s>\n",
      "prompt: ['<s>', 'unge', 'norske']\n",
      "['<s>', 'unge', 'norske', 'mann', '</s>']\n",
      "---\n",
      "---\n",
      "train sent:\n",
      " <s> har du seet hva som stiger </s>\n",
      "prompt: ['<s>', 'har', 'du']\n",
      "['<s>', 'har', 'du', 'hørt', 'hva', 'som', 'stiger', '</s>']\n",
      "---\n"
     ]
    }
   ],
   "source": [
    "for sentence in sentences[1:]:\n",
    "    print(\"---\")\n",
    "    print(\"train sent:\\n\",\" \".join(sentence))\n",
    "    print(\"prompt:\", sentence[:3])\n",
    "    print(generate_next(\" \".join(sentence[:3]), model, word_model.wv, num_generated=10, temperature=0, eos=eos))\n",
    "    print(\"---\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "10d64fec",
   "metadata": {},
   "source": [
    "# Sanity stanza level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e5a64b6d",
   "metadata": {},
   "outputs": [],
   "source": [
    "pad = \"<pad>\"\n",
    "sos = \"<s>\"\n",
    "eos = \"</s>\"\n",
    "newline = \"<n>\"\n",
    "\n",
    "stanza = get_stanzas(\"../../norwegian_rhyme_scheme_corpus/poems/bokmål/\", pad, sos, eos, newline)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "399b9915",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['<pad>'],\n",
       " ['<s>',\n",
       "  'svar',\n",
       "  'fra',\n",
       "  'norge',\n",
       "  '<n>',\n",
       "  'har',\n",
       "  'du',\n",
       "  'hørt',\n",
       "  'hva',\n",
       "  'svensken',\n",
       "  'sier',\n",
       "  '<n>',\n",
       "  'unge',\n",
       "  'norske',\n",
       "  'mann',\n",
       "  '<n>',\n",
       "  'har',\n",
       "  'du',\n",
       "  'seet',\n",
       "  'hva',\n",
       "  'som',\n",
       "  'stiger',\n",
       "  '<n>',\n",
       "  'opp',\n",
       "  'om',\n",
       "  'kjølens',\n",
       "  'rand',\n",
       "  '<n>',\n",
       "  '</s>'],\n",
       " ['<s>',\n",
       "  'skygger',\n",
       "  'av',\n",
       "  'de',\n",
       "  'falne',\n",
       "  'fedre',\n",
       "  '<n>',\n",
       "  'som',\n",
       "  'har',\n",
       "  'aldri',\n",
       "  'visst',\n",
       "  'det',\n",
       "  'bedre',\n",
       "  '<n>',\n",
       "  'enn',\n",
       "  'hvor',\n",
       "  'slike',\n",
       "  'ord',\n",
       "  'ble',\n",
       "  'sagt',\n",
       "  '<n>',\n",
       "  'der',\n",
       "  'å',\n",
       "  'vinke',\n",
       "  'frem',\n",
       "  'til',\n",
       "  'vakt',\n",
       "  '<n>',\n",
       "  '</s>'],\n",
       " ['<s>',\n",
       "  'svensken',\n",
       "  'sier',\n",
       "  'at',\n",
       "  'det',\n",
       "  'røde',\n",
       "  '<n>',\n",
       "  'i',\n",
       "  'vårt',\n",
       "  'norske',\n",
       "  'flagg',\n",
       "  '<n>',\n",
       "  'det',\n",
       "  'som',\n",
       "  'rant',\n",
       "  'da',\n",
       "  'magnus',\n",
       "  'døde',\n",
       "  '<n>',\n",
       "  'det',\n",
       "  'som',\n",
       "  'ler',\n",
       "  'i',\n",
       "  'dag',\n",
       "  '<n>',\n",
       "  'det',\n",
       "  'som',\n",
       "  'over',\n",
       "  'halden',\n",
       "  'beltet',\n",
       "  '<n>',\n",
       "  'det',\n",
       "  'som',\n",
       "  'over',\n",
       "  'adler',\n",
       "  'veltet',\n",
       "  '<n>',\n",
       "  'det',\n",
       "  'kan',\n",
       "  'svenskens',\n",
       "  'gule-blå',\n",
       "  '<n>',\n",
       "  'uten',\n",
       "  'skam',\n",
       "  'ei',\n",
       "  'bære',\n",
       "  'på',\n",
       "  '<n>',\n",
       "  '</s>'],\n",
       " ['<s>',\n",
       "  'svensken',\n",
       "  'sier',\n",
       "  'våre',\n",
       "  'minner',\n",
       "  '<n>',\n",
       "  'have',\n",
       "  'tapt',\n",
       "  'sin',\n",
       "  'glans',\n",
       "  '<n>',\n",
       "  'at',\n",
       "  'vi',\n",
       "  'ære',\n",
       "  'lettest',\n",
       "  'finner',\n",
       "  '<n>',\n",
       "  'ved',\n",
       "  'å',\n",
       "  'låne',\n",
       "  'hans',\n",
       "  '<n>',\n",
       "  '</s>']]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stanzas = stanza[:5]\n",
    "stanzas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1cc785be",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocab size: 81\n",
      "Embedding size: 100\n"
     ]
    }
   ],
   "source": [
    "word_model = train_word2vec(stanzas)\n",
    "pretrained_weights = word_model.wv.vectors\n",
    "vocab_size, embedding_size = pretrained_weights.shape\n",
    "print(f\"Vocab size: {vocab_size}\\nEmbedding size: {embedding_size}\")\n",
    "\n",
    "pad_idx = word_model.wv.key_to_index[pad]\n",
    "assert pad_idx == 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1b3fc8cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = get_train_data(stanzas, word_model.wv, pad_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b16b2890",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((126, 48), (126,))"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5a3bdf7c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-05-03 18:18:04.583471: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory\n",
      "2022-05-03 18:18:04.583497: W tensorflow/stream_executor/cuda/cuda_driver.cc:269] failed call to cuInit: UNKNOWN ERROR (303)\n",
      "2022-05-03 18:18:04.583515: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (tita-laptop): /proc/driver/nvidia/version does not exist\n",
      "2022-05-03 18:18:04.584813: I tensorflow/core/platform/cpu_feature_guard.cc:151] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "model_name = \"sanity_stanza\"\n",
    "\n",
    "# # uncomment to train model\n",
    "# print('\\nTraining LSTM...')\n",
    "# model = Sequential([\n",
    "#         Embedding(input_dim=vocab_size, output_dim=embedding_size, weights=[pretrained_weights], mask_zero=True),\n",
    "#         LSTM(units=embedding_size),\n",
    "#         Dense(units=vocab_size),\n",
    "#         Activation('softmax')\n",
    "#         ])\n",
    "\n",
    "\n",
    "# model.compile(optimizer='adam', loss='sparse_categorical_crossentropy')\n",
    "\n",
    "# model_checkpoint = ModelCheckpoint(f\"models/{model_name}.hdf5\",monitor=\"val_loss\")\n",
    "# terminate_on_nan = TerminateOnNaN()\n",
    "# csv_logger = CSVLogger(f'training_{model_name}.log')\n",
    "# history = model.fit(X, y,\n",
    "#           batch_size=1,\n",
    "#           epochs=20,\n",
    "#           callbacks=[model_checkpoint, terminate_on_nan])\n",
    "\n",
    "model = load_model(f\"models/{model_name}.hdf5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "34843995",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---\n",
      "train sent:\n",
      "<s> svar fra norge <n> har du hørt hva svensken sier <n> unge norske mann <n> har du seet hva som stiger <n> opp om kjølens rand <n> </s>\n",
      "\n",
      "prompt:['<s>', 'svar', 'fra', 'norge']\n",
      "\n",
      " svar fra norge \n",
      " har du hørt hva svensken sier \n",
      " unge norske mann \n",
      " har du seet hva som stiger \n",
      " opp om kjølens rand \n",
      " \n",
      "---\n",
      "---\n",
      "train sent:\n",
      "<s> skygger av de falne fedre <n> som har aldri visst det bedre <n> enn hvor slike ord ble sagt <n> der å vinke frem til vakt <n> </s>\n",
      "\n",
      "prompt:['<s>', 'skygger', 'av', 'de']\n",
      "\n",
      " skygger av de falne fedre \n",
      " som har aldri visst det bedre \n",
      " enn hvor slike ord ble sagt \n",
      " der å vinke frem til vakt \n",
      " \n",
      "---\n",
      "---\n",
      "train sent:\n",
      "<s> svensken sier at det røde <n> i vårt norske flagg <n> det som rant da magnus døde <n> det som ler i dag <n> det som over halden beltet <n> det som over adler veltet <n> det kan svenskens gule-blå <n> uten skam ei bære på <n> </s>\n",
      "\n",
      "prompt:['<s>', 'svensken', 'sier', 'at']\n",
      "\n",
      " svensken sier at det røde \n",
      " i vårt norske flagg \n",
      " det som rant da magnus døde \n",
      " det som ler i dag \n",
      " det som over halden beltet \n",
      " det som over adler veltet \n",
      " det kan svenskens gule-blå \n",
      " uten skam ei bære på \n",
      " \n",
      "---\n",
      "---\n",
      "train sent:\n",
      "<s> svensken sier våre minner <n> have tapt sin glans <n> at vi ære lettest finner <n> ved å låne hans <n> </s>\n",
      "\n",
      "prompt:['<s>', 'svensken', 'sier', 'våre']\n",
      "\n",
      " svensken sier våre minner \n",
      " have tapt sin glans \n",
      " at vi ære lettest finner \n",
      " ved å låne hans \n",
      " \n",
      "---\n"
     ]
    }
   ],
   "source": [
    "for stanza in stanzas[1:]:\n",
    "    print(\"---\")\n",
    "    print(f\"train sent:\\n{' '.join(stanza)}\\n\")\n",
    "    print(f\"prompt:{stanza[:4]}\\n\")\n",
    "    gen = generate_next(\" \".join(stanza[:4]), model, word_model.wv, num_generated=100, temperature=0, eos=eos)\n",
    "    print(stanza_to_string(gen, sos, eos, newline))\n",
    "    print(\"---\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "66308b98",
   "metadata": {},
   "source": [
    "# Sanity reverse stanza level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "a10b49b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "pad = \"<pad>\"\n",
    "sos = \"<s>\"\n",
    "eos = \"</s>\"\n",
    "newline = \"<n>\"\n",
    "\n",
    "stanzas = get_stanzas(\"../../norwegian_rhyme_scheme_corpus/poems/bokmål/\", pad, sos, eos, newline)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "3e4e6f93",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['<pad>'],\n",
       " ['</s>',\n",
       "  '<n>',\n",
       "  'rand',\n",
       "  'kjølens',\n",
       "  'om',\n",
       "  'opp',\n",
       "  '<n>',\n",
       "  'stiger',\n",
       "  'som',\n",
       "  'hva',\n",
       "  'seet',\n",
       "  'du',\n",
       "  'har',\n",
       "  '<n>',\n",
       "  'mann',\n",
       "  'norske',\n",
       "  'unge',\n",
       "  '<n>',\n",
       "  'sier',\n",
       "  'svensken',\n",
       "  'hva',\n",
       "  'hørt',\n",
       "  'du',\n",
       "  'har',\n",
       "  '<n>',\n",
       "  'norge',\n",
       "  'fra',\n",
       "  'svar',\n",
       "  '<s>'],\n",
       " ['</s>',\n",
       "  '<n>',\n",
       "  'vakt',\n",
       "  'til',\n",
       "  'frem',\n",
       "  'vinke',\n",
       "  'å',\n",
       "  'der',\n",
       "  '<n>',\n",
       "  'sagt',\n",
       "  'ble',\n",
       "  'ord',\n",
       "  'slike',\n",
       "  'hvor',\n",
       "  'enn',\n",
       "  '<n>',\n",
       "  'bedre',\n",
       "  'det',\n",
       "  'visst',\n",
       "  'aldri',\n",
       "  'har',\n",
       "  'som',\n",
       "  '<n>',\n",
       "  'fedre',\n",
       "  'falne',\n",
       "  'de',\n",
       "  'av',\n",
       "  'skygger',\n",
       "  '<s>'],\n",
       " ['</s>',\n",
       "  '<n>',\n",
       "  'på',\n",
       "  'bære',\n",
       "  'ei',\n",
       "  'skam',\n",
       "  'uten',\n",
       "  '<n>',\n",
       "  'gule-blå',\n",
       "  'svenskens',\n",
       "  'kan',\n",
       "  'det',\n",
       "  '<n>',\n",
       "  'veltet',\n",
       "  'adler',\n",
       "  'over',\n",
       "  'som',\n",
       "  'det',\n",
       "  '<n>',\n",
       "  'beltet',\n",
       "  'halden',\n",
       "  'over',\n",
       "  'som',\n",
       "  'det',\n",
       "  '<n>',\n",
       "  'dag',\n",
       "  'i',\n",
       "  'ler',\n",
       "  'som',\n",
       "  'det',\n",
       "  '<n>',\n",
       "  'døde',\n",
       "  'magnus',\n",
       "  'da',\n",
       "  'rant',\n",
       "  'som',\n",
       "  'det',\n",
       "  '<n>',\n",
       "  'flagg',\n",
       "  'norske',\n",
       "  'vårt',\n",
       "  'i',\n",
       "  '<n>',\n",
       "  'røde',\n",
       "  'det',\n",
       "  'at',\n",
       "  'sier',\n",
       "  'svensken',\n",
       "  '<s>'],\n",
       " ['</s>',\n",
       "  '<n>',\n",
       "  'hans',\n",
       "  'låne',\n",
       "  'å',\n",
       "  'ved',\n",
       "  '<n>',\n",
       "  'finner',\n",
       "  'lettest',\n",
       "  'ære',\n",
       "  'vi',\n",
       "  'at',\n",
       "  '<n>',\n",
       "  'glans',\n",
       "  'sin',\n",
       "  'tapt',\n",
       "  'have',\n",
       "  '<n>',\n",
       "  'minner',\n",
       "  'våre',\n",
       "  'sier',\n",
       "  'svensken',\n",
       "  '<s>']]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stanzas = stanzas[:5]\n",
    "for s in stanzas:\n",
    "    s.reverse()\n",
    "stanzas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "e72292ba",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocab size: 81\n",
      "Embedding size: 100\n"
     ]
    }
   ],
   "source": [
    "word_model = train_word2vec(stanzas)\n",
    "pretrained_weights = word_model.wv.vectors\n",
    "vocab_size, embedding_size = pretrained_weights.shape\n",
    "print(f\"Vocab size: {vocab_size}\\nEmbedding size: {embedding_size}\")\n",
    "\n",
    "pad_idx = word_model.wv.key_to_index[pad]\n",
    "assert pad_idx == 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "8707a96a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((126, 48), (126,))"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X, y = get_train_data(stanzas, word_model.wv, pad_idx)\n",
    "X.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "0bee4986",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model_name = \"sanity_reverse_stanza\"\n",
    "\n",
    "# # uncomment to train model\n",
    "# print('\\nTraining LSTM...')\n",
    "# model = Sequential([\n",
    "#         Embedding(input_dim=vocab_size, output_dim=embedding_size, weights=[pretrained_weights], mask_zero=True),\n",
    "#         LSTM(units=embedding_size),\n",
    "#         Dense(units=vocab_size),\n",
    "#         Activation('softmax')\n",
    "#         ])\n",
    "\n",
    "\n",
    "# model.compile(optimizer='adam', loss='sparse_categorical_crossentropy')\n",
    "\n",
    "# model_checkpoint = ModelCheckpoint(f\"models/{model_name}.hdf5\",monitor=\"val_loss\")\n",
    "# terminate_on_nan = TerminateOnNaN()\n",
    "# csv_logger = CSVLogger(f'training_{model_name}.log')\n",
    "# history = model.fit(X, y,\n",
    "#           batch_size=1,\n",
    "#           epochs=20,\n",
    "#           callbacks=[model_checkpoint, terminate_on_nan])\n",
    "\n",
    "model = load_model(f\"models/{model_name}.hdf5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "eefc58ea",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---\n",
      "train sent (reverse):\n",
      "</s> <n> rand kjølens om opp <n> stiger som hva seet du har <n> mann norske unge <n> sier svensken hva hørt du har <n> norge fra svar <s>\n",
      "\n",
      "prompt:['</s>', '<n>', 'rand', 'kjølens']\n",
      "\n",
      " svar fra norge \n",
      " har du hørt hva svensken sier \n",
      " unge norske mann \n",
      " har du seet hva som stiger \n",
      " opp om kjølens rand \n",
      " \n",
      "---\n",
      "---\n",
      "train sent (reverse):\n",
      "</s> <n> vakt til frem vinke å der <n> sagt ble ord slike hvor enn <n> bedre det visst aldri har som <n> fedre falne de av skygger <s>\n",
      "\n",
      "prompt:['</s>', '<n>', 'vakt', 'til']\n",
      "\n",
      " skygger av de falne fedre \n",
      " som har aldri visst det bedre \n",
      " enn hvor slike ord ble sagt \n",
      " der å vinke frem til vakt \n",
      " \n",
      "---\n",
      "---\n",
      "train sent (reverse):\n",
      "</s> <n> på bære ei skam uten <n> gule-blå svenskens kan det <n> veltet adler over som det <n> beltet halden over som det <n> dag i ler som det <n> døde magnus da rant som det <n> flagg norske vårt i <n> røde det at sier svensken <s>\n",
      "\n",
      "prompt:['</s>', '<n>', 'på', 'bære']\n",
      "\n",
      " svensken sier at det røde \n",
      " i vårt norske flagg \n",
      " det som rant da magnus døde \n",
      " det som over halden beltet \n",
      " det som over adler veltet \n",
      " det kan svenskens gule-blå \n",
      " uten skam ei bære på \n",
      " \n",
      "---\n",
      "---\n",
      "train sent (reverse):\n",
      "</s> <n> hans låne å ved <n> finner lettest ære vi at <n> glans sin tapt have <n> minner våre sier svensken <s>\n",
      "\n",
      "prompt:['</s>', '<n>', 'hans', 'låne']\n",
      "\n",
      " svensken sier våre minner \n",
      " have tapt sin glans \n",
      " at vi ære lettest finner \n",
      " ved å låne hans \n",
      " \n",
      "---\n"
     ]
    }
   ],
   "source": [
    "for stanza in stanzas[1:]:\n",
    "    print(\"---\")\n",
    "    print(f\"train sent (reverse):\\n{' '.join(stanza)}\\n\")\n",
    "    print(f\"prompt:{stanza[:4]}\\n\")\n",
    "    gen = generate_next(\" \".join(stanza[:4]), model, word_model.wv, num_generated=100, temperature=0, eos=sos, reverse=True)\n",
    "    print(stanza_to_string(gen, sos, eos, newline))\n",
    "    print(\"---\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35076e21",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "108cee79",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c0d9fe6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da7561a8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b38873cf",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "a3aa09aa",
   "metadata": {},
   "source": [
    "# Full stanza level LSTM\n",
    "(ended up not using this)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "aaa4000a",
   "metadata": {},
   "outputs": [],
   "source": [
    "pad = \"<pad>\"\n",
    "sos = \"<s>\"\n",
    "eos = \"</s>\"\n",
    "newline = \"<n>\"\n",
    "\n",
    "stanzas = get_stanzas(\"../../norwegian_rhyme_scheme_corpus/poems/bokmål/\", pad, sos, eos, newline)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cc88a135",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocab size: 17867\n",
      "Embedding_size: 100\n"
     ]
    }
   ],
   "source": [
    "model_name = \"poetry_gen_stanza\"\n",
    "# # Uncomment to train\n",
    "# word_model = train_word2vec(stanzas)\n",
    "# word_model.save(f\"models/word2vec_{model_name}.model\")\n",
    "\n",
    "word_model = gensim.models.Word2Vec.load(f\"models/word2vec_{model_name}.model\")\n",
    "\n",
    "pretrained_weights = word_model.wv.vectors\n",
    "vocab_size, embedding_size = pretrained_weights.shape\n",
    "print(f\"Vocab size: {vocab_size}\\nEmbedding_size: {embedding_size}\")\n",
    "pad_idx = word_model.wv.key_to_index[pad]\n",
    "assert pad_idx == 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "46523f31",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((185896, 119), (185896,))"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X, y = get_train_data(stanzas, word_model.wv, pad_idx)\n",
    "X.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "20e737d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_dev, y_train, y_dev = train_test_split(X, y, test_size=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b4d62d93",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((184896, 119), (1000, 119))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape, X_dev.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "760bd6f9",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-05-04 02:24:43.880703: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory\n",
      "2022-05-04 02:24:43.880745: W tensorflow/stream_executor/cuda/cuda_driver.cc:269] failed call to cuInit: UNKNOWN ERROR (303)\n",
      "2022-05-04 02:24:43.880771: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (tita-laptop): /proc/driver/nvidia/version does not exist\n",
      "2022-05-04 02:24:43.881098: I tensorflow/core/platform/cpu_feature_guard.cc:151] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "model_name = \"poetry_gen_stanza\"\n",
    "\n",
    "# # uncomment to train model\n",
    "# print('\\nTraining LSTM...')\n",
    "# model = Sequential([\n",
    "#         Embedding(input_dim=vocab_size, output_dim=embedding_size, weights=[pretrained_weights], mask_zero=True),\n",
    "#         LSTM(units=embedding_size),\n",
    "#         Dense(units=vocab_size),\n",
    "#         Activation('softmax')\n",
    "#         ])\n",
    "\n",
    "\n",
    "# model.compile(optimizer='adam', loss='sparse_categorical_crossentropy')\n",
    "\n",
    "# model_checkpoint = tf.keras.callbacks.ModelCheckpoint(f\"models/{model_name}.hdf5\",monitor=\"val_loss\")\n",
    "# terminate_on_nan = tf.keras.callbacks.TerminateOnNaN()\n",
    "# csv_logger = tf.keras.callbacks.CSVLogger(f'logs/training_{model_name}.log')\n",
    "# early_stop = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=5)\n",
    "\n",
    "# history = model.fit(X_train, y_train,\n",
    "#                     batch_size=128,\n",
    "#                     epochs=100,\n",
    "#                     validation_data=(X_dev, y_dev),\n",
    "#                     callbacks=[model_checkpoint, terminate_on_nan, csv_logger, early_stop])\n",
    "\n",
    "model = load_model(f\"models/{model_name}.hdf5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "03fa0dab",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "prompt: <s> ser du\n",
      " ser du barn av latter \n",
      " helt vet jeg mange og grå er og godt \n",
      " og byen når man vokser at hjemme \n",
      " og får man lyse hus fler hit \n",
      " si er det er mere blitt sang så god \n",
      " er du en mann til sist oss husk \n",
      " at du skulle krenke det \n",
      " \n",
      "---\n",
      "prompt: <s> har du\n",
      " har du sett deg disse stener \n",
      " ditt fang \n",
      " revet av et spill \n",
      " under disse stolte livet \n",
      " med stolte under all trengsel \n",
      " trange gylne slott for lysredd og for nye mot \n",
      " for verdens glans og i eie \n",
      " å folk den skjulte stad \n",
      " \n",
      "---\n",
      "prompt: <s> for\n",
      " for en fordomsfri fortolkning \n",
      " synes sakens løsning funnet \n",
      " visselig er dagen rundet \n",
      " for den ganske jords befolkning \n",
      " klare ligger herrens planer \n",
      " dog foruten makt den makt og spilt \n",
      " \n",
      "---\n",
      "prompt: <s> jeg vil\n",
      " jeg vil fare ut i all min velde \n",
      " for dette slag er din vinning din sjel \n",
      " vil jeg deg gud bring din vrede \n",
      " o hvor såre hvorfor fant ham \n",
      " jeg sviktet hvorfor å føre dere venner \n",
      " for henne og herre og herre lever \n",
      " opfylie kan vi tapte fremmede guder \n",
      " min vugge en dag hvem stod jeg slår \n",
      " og min sønn min sønn har herren vår \n",
      " \n",
      "---\n"
     ]
    }
   ],
   "source": [
    "sample_sents = [\n",
    "    \"<s> ser du\",\n",
    "    \"<s> har du\",\n",
    "    \"<s> for\",\n",
    "    \"<s> jeg vil\",\n",
    "]\n",
    "\n",
    "for prompt in sample_sents:\n",
    "    print(f\"prompt: {prompt}\")\n",
    "    gen = generate_next(prompt, model, word_model.wv, num_generated=100, temperature=0, eos=eos)\n",
    "    print(stanza_to_string(gen, sos, eos, newline))\n",
    "    print(\"---\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}