nbdigital/Extract_poetry_and_compare_re-OCR_batches.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract poems based on filename"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of files: 21483\n",
      "Number of files that match with pattern 0\n"
     ]
    }
   ],
   "source": [
    "import tarfile\n",
    "import re\n",
    "\n",
    "tar = tarfile.open('20150526_nbdig_txt01.tar.gz')\n",
    "\n",
    "pattern = r\".*[dD]i[g|k]t.*|.*[lL]yrik.*|.*[Ss]alme.*|.*[Ss]ang.*\"\n",
    "\n",
    "i = 0\n",
    "j = 0\n",
    "\n",
    "poems = []\n",
    "\n",
    "for file in tar:\n",
    "    if file.isfile():\n",
    "        j += 1\n",
    "        if re.match(pattern, file.name):\n",
    "            i += 1\n",
    "            poems.append(file)\n",
    "            \n",
    "print(f\"Total number of files: {j}\\nNumber of files that match with pattern {i}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "for e in poems:\n",
    "    tar.extract(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "dirname = \"idetfrikorpus-dist-txt/\"\n",
    "p = Path(dirname)\n",
    "\n",
    "txts = [e for e in p.iterdir() if e.suffix == \".txt\"]\n",
    "dirs = [e for e in p.iterdir() if e.is_dir()]\n",
    "\n",
    "txts += [t for d in dirs for t in d.iterdir()]\n",
    "\n",
    "poems = txts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Find out how many of the 593 where the title matches (not filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json \n",
    "import requests\n",
    "\n",
    "def get_code(filename):\n",
    "    return filename.split(\"_\")[1].split(\"-\")[0]\n",
    "\n",
    "def get_metadata(codes):\n",
    "    metadata = {}\n",
    "    base_api = \"https://api.nb.no/catalog/v1/items/URN:NBN:no-nb_digibok_\"\n",
    "    for code in codes:\n",
    "        response = requests.get(base_api+code)\n",
    "        j = response.json()\n",
    "        metadata[code] = j[\"metadata\"]\n",
    "    return metadata\n",
    "\n",
    "# # uncomment this to call api (commexnted out to not run unecessarily many times)\n",
    "# codes = [get_code(poem.name) for poem in poems]\n",
    "# metadata = get_metadata(codes)\n",
    "\n",
    "# with open(\"metadata_593.json\", \"w+\") as f:\n",
    "#     json.dump(metadata, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Received code: 2008042303004, metadata code: 2021120826007\n",
      "Received code: 2009013010001, metadata code: 2010050510003\n",
      "Received code: 2012011012011, metadata code: 2012011712004\n",
      "Received code: 2008102303002, metadata code: 2015071427011\n",
      "Received code: 2012011012013, metadata code: 2012011712005\n",
      "Received code: 2006111600061, metadata code: 2017011248051\n",
      "Received code: 2008010704054, metadata code: 2014102306006\n",
      "Received code: 2012101224024, metadata code: 2012101224027\n",
      "Received code: 2009040212006, metadata code: 2009040212008\n",
      "Received code: 2007121410004, metadata code: 2008040800024\n",
      "Received code: 2012030212009, metadata code: 2013032526002\n",
      "Received code: 2008071410004, metadata code: 2014071728003\n",
      "Received code: 2008071610001, metadata code: 2019120928001\n",
      "Received code: 2009051513004, metadata code: 2011032412001\n",
      "Received code: 2013050224034, metadata code: 2013050224102\n",
      "Received code: 2010040912003, metadata code: 2017020748579\n",
      "Received code: 2010041612002, metadata code: 2010041612003\n",
      "Received code: 2010040912004, metadata code: 2010041212002\n",
      "Received code: 2013050224027, metadata code: 2013050224104\n",
      "Received code: 2013052824002, metadata code: 2014080424009\n",
      "Received code: 2009032603016, metadata code: 2010060420012\n",
      "Received code: 2010040912002, metadata code: 2017030148004\n",
      "Received code: 2010011312016, metadata code: 2010011312017\n",
      "Received code: 2010051412002, metadata code: 2010060420002\n",
      "Received code: 2010011312007, metadata code: 2010011312008\n",
      "Received code: 2010042712002, metadata code: 2010042812001\n",
      "Received code: 2008121603021, metadata code: 2016111748604\n",
      "Received code: 2012051113007, metadata code: 2014031028002\n",
      "Received code: 2007082703002, metadata code: 2020022648153\n",
      "Received code: 2008061913007, metadata code: 2008061913008\n",
      "Received code: 2008060613027, metadata code: 2008060613028\n",
      "Received code: 2008061913009, metadata code: 2008061913011\n",
      "Received code: 2011012412003, metadata code: 2018062148507\n",
      "Received code: 2009072713006, metadata code: 2009072713007\n",
      "Received code: 2011110724021, metadata code: 2011110724022\n",
      "Received code: 2013040824027, metadata code: 2015111826003\n",
      "Received code: 2007073010003, metadata code: 2011041820007\n",
      "Received code: 2006120701085, metadata code: 2007080110001\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "def metadata_to_df(metadata):\n",
    "    d = []\n",
    "    for code, data in metadata.items():\n",
    "        lang = data[\"languages\"][0][\"code\"]\n",
    "        year = int(data[\"originInfo\"][\"issued\"])\n",
    "        title = data[\"title\"]\n",
    "        \n",
    "        data_code = data[\"identifiers\"][\"urn\"].split(\"_\")[-1]\n",
    "        if code != data_code:\n",
    "            print(f\"Received code: {code}, metadata code: {data_code}\")\n",
    "\n",
    "        if \"people\" in data:\n",
    "            author = data[\"people\"][0][\"name\"]\n",
    "        elif \"statementOfResponsibility\" in data:\n",
    "            author = data[\"statementOfResponsibility\"][0]\n",
    "        d.append({\"year\": year, \"author\": author, \"title\": title, \"filename code\": code, \"api code\": data_code, \"lang\": lang})\n",
    "    return pd.DataFrame(d)\n",
    "\n",
    "with open(\"metadata_593.json\") as f:\n",
    "    metadata = json.load(f)\n",
    "\n",
    "df_593 = metadata_to_df(metadata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "      <th>filename code</th>\n",
       "      <th>api code</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1830</td>\n",
       "      <td>Guldberg, O. Høegh (Ove)</td>\n",
       "      <td>Psalmebog, eller En Samling af gamle og nye Ps...</td>\n",
       "      <td>2010063003020</td>\n",
       "      <td>2010063003020</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1869</td>\n",
       "      <td>Guldberg, O. Høegh (Ove)</td>\n",
       "      <td>Fra nordiske Digtere : et Album</td>\n",
       "      <td>2006112101006</td>\n",
       "      <td>2006112101006</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1862</td>\n",
       "      <td>Barfod, Frederik</td>\n",
       "      <td>Ingemanns Jordefærd : Taler, Digte og Bladarti...</td>\n",
       "      <td>2009090403002</td>\n",
       "      <td>2009090403002</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1856</td>\n",
       "      <td>Barfod, Frederik</td>\n",
       "      <td>Guds Man : en Psalme-Bog</td>\n",
       "      <td>2006111600006</td>\n",
       "      <td>2006111600006</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1834</td>\n",
       "      <td>Grøndahl, Chr. (Christopher)</td>\n",
       "      <td>Evangelisk-christelig Psalmebog, tilligemed Co...</td>\n",
       "      <td>2008051403001</td>\n",
       "      <td>2008051403001</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>588</th>\n",
       "      <td>1925</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Stene i strømmen : digte</td>\n",
       "      <td>2013070308043</td>\n",
       "      <td>2013070308043</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>589</th>\n",
       "      <td>1947</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Samlede dikt</td>\n",
       "      <td>2012082408040</td>\n",
       "      <td>2012082408040</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>590</th>\n",
       "      <td>1950</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Samlede dikt</td>\n",
       "      <td>2011102708060</td>\n",
       "      <td>2011102708060</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>591</th>\n",
       "      <td>1912</td>\n",
       "      <td>Munthe, Margrethe</td>\n",
       "      <td>Aase fiskerpike : eventyrkomedie med sange til...</td>\n",
       "      <td>2013073008139</td>\n",
       "      <td>2013073008139</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>592</th>\n",
       "      <td>1862</td>\n",
       "      <td>Johnsen, J.</td>\n",
       "      <td>En aandelig Sangbog for angerfulde, bodfærdige...</td>\n",
       "      <td>2010092803011</td>\n",
       "      <td>2010092803011</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>593 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     year                        author  \\\n",
       "0    1830      Guldberg, O. Høegh (Ove)   \n",
       "1    1869      Guldberg, O. Høegh (Ove)   \n",
       "2    1862              Barfod, Frederik   \n",
       "3    1856              Barfod, Frederik   \n",
       "4    1834  Grøndahl, Chr. (Christopher)   \n",
       "..    ...                           ...   \n",
       "588  1925                Grieg, Nordahl   \n",
       "589  1947                Grieg, Nordahl   \n",
       "590  1950                Grieg, Nordahl   \n",
       "591  1912             Munthe, Margrethe   \n",
       "592  1862                   Johnsen, J.   \n",
       "\n",
       "                                                 title  filename code  \\\n",
       "0    Psalmebog, eller En Samling af gamle og nye Ps...  2010063003020   \n",
       "1                      Fra nordiske Digtere : et Album  2006112101006   \n",
       "2    Ingemanns Jordefærd : Taler, Digte og Bladarti...  2009090403002   \n",
       "3                             Guds Man : en Psalme-Bog  2006111600006   \n",
       "4    Evangelisk-christelig Psalmebog, tilligemed Co...  2008051403001   \n",
       "..                                                 ...            ...   \n",
       "588                           Stene i strømmen : digte  2013070308043   \n",
       "589                                       Samlede dikt  2012082408040   \n",
       "590                                       Samlede dikt  2011102708060   \n",
       "591  Aase fiskerpike : eventyrkomedie med sange til...  2013073008139   \n",
       "592  En aandelig Sangbog for angerfulde, bodfærdige...  2010092803011   \n",
       "\n",
       "          api code lang  \n",
       "0    2010063003020  dan  \n",
       "1    2006112101006  dan  \n",
       "2    2009090403002  dan  \n",
       "3    2006111600006  nob  \n",
       "4    2008051403001  dan  \n",
       "..             ...  ...  \n",
       "588  2013070308043  nob  \n",
       "589  2012082408040  nob  \n",
       "590  2011102708060  nob  \n",
       "591  2013073008139  nob  \n",
       "592  2010092803011  nob  \n",
       "\n",
       "[593 rows x 6 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_593"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Count files where where codes differ and compare titles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(38, 38)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d1 = set(df_593[\"api code\"]) - set(df_593[\"filename code\"])\n",
    "d2 = set(df_593[\"filename code\"]) - set(df_593[\"api code\"])\n",
    "len(d1), len(d2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "diff_code_poems = [e.name for e in poems if get_code(e.name) in d2]\n",
    "\n",
    "poem_codes = [get_code(e) for e in diff_code_poems]\n",
    "poem_titles = [\" \".join(e.split(\"-\")[5][:-4].split(\"_\")) for e in diff_code_poems]\n",
    "poem_authors = [e.split(\"-\")[4] for e in diff_code_poems]\n",
    "\n",
    "check1_df = pd.DataFrame({\"title\":poem_titles, \"author\":poem_authors, \"filename code\": poem_codes })\n",
    "#check1_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "check2_df = df_593.loc[df_593[\"api code\"].isin(d1)]\n",
    "#check2_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Looks like they are the same!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Psalme Skat et Udvalg af gamle Psalmer uddrag\n",
      "Psalme-Skat : Et Udvalg af gamle Psalmer, uddraget af Kingos, Pontoppidans og Brorsons Samlinger\n",
      "\n",
      "En Ny Danske oc Norske fuldkommen Psalme Bog \n",
      "En Ny Danske oc Norske fuldkommen Psalme-Bog : hvor udi findis et Tusinde Psalmer oc Sange, hvilcke med største Flijd ere samlede af Mester Hans Thomesøns, samt alleandre fornemste gamle oc beste nye, tryckte Danske aandeligePsalme-Bøger\n",
      "\n",
      "Melodier til Sangene i Lesebog for Folkeskole\n",
      "Melodier til Sangene i Læsebog for Folkeskolen og Folkehjemmet : udsatte for Sopran, Alt og Bass, tilligemed en Tenor efter Behag, : for Skoler og Sangforeninger : 1. Skoletrin\n",
      "\n",
      "Evangelisk christelig Psalmebog tilligemed Co\n",
      "Evangelisk-christelig Psalmebog, tilligemed Collecter, Epistler og Evangelier, Vor Herres Jesu Christi Lidelses-Historie, Kirke- samt andre Bønner, til Brug ved Kirke- og Huus-Andagt\n",
      "\n",
      "Melodier til Sangene i Lesebog for Folkeskole\n",
      "Melodier til Sangene i Læsebog for Folkeskolen og Folkehjemmet : udsatte for Sopran, Alt og Bass, tilligemed en Tenor efter Behag, : for Skoler og Sangforeninger : 3. Skoletrin\n",
      "\n",
      "Oversigt over Kirkens Salmesang dens Historie\n",
      "Oversigt over Kirkens Salmesang, dens Historie og om Kirkesalmebøgerne i de nordiske Lande\n",
      "\n",
      "Utvalgte dikt \n",
      "Utvalgte dikt\n",
      "\n",
      "Udvalg af eldre danske Digteres Verker \n",
      "Udvalg af ældre danske Digteres Værker. 2 H. 1 : Holberg Den politiske Kandestøber ; Barselstuen\n",
      "\n",
      "Den heldige Beslutning en Sang paa veledle ho\n",
      "Den hældige Beslutning : en Sang paa velædle, høyfornemme Hr. Friderich Hvids og velædle, høyfornemme Mad. Maria Christine Olsens Bryllupsfest den 24de Februarii 1801\n",
      "\n",
      "Den store Freden dramatisk Dikt\n",
      "Den store Freden : dramatisk Dikt\n",
      "\n",
      "Taler og Sange i Aggershuus Slotskirke paa de\n",
      "Taler og Sange i Aggershuus Slotskirke paa den anordnede høitidelige Bededag, den 25. Februar 1814\n",
      "\n",
      "Sielens Sang Offer Gudelige Sange paa de forn\n",
      "Siælens Sang-Offer. Gudelige Sange paa de fornemste Fæster om Syndernis Forladelse, Fortrøstning paa Guds Naade, mod Fortviflelse, Utaalmodighed, og saa fremdelis: med Afften og Morgen-Sang i Tallet 42 Morgen og Afften-Suk til hver Dag i Ugen, slutted med en Morgen og Afften Bøn. Enfoldig ved Guds Naade sammensat aff den Der Er Dig Høye Himmel-Gud som sin Siælis Brudgom, all Ære og Tieneste Skyldig\n",
      "\n",
      "Dorothe Engelbrets Datters Aandelige Sang og \n",
      "Dorothe Engelbrets-Datters Aandelige Sang- og Taare-Offer\n",
      "\n",
      "Sielens Sang Offer Indeholdende Gudelige Sang\n",
      "Siælens Sang-Offer Indeholdende Gudelige Sange paa de Fornemste Fester, tillige med andre sær Himmelske Sange, saa og om Syndernis Forladelse og Fortrøstning paa Guds Naade Mod Fortuilelse og Utaalmodighed, Verdens Omskiftelse og de Fortrædeligis Tunger: Trøst mod Døden og Dommen, og Glæden for de Vdvalde effter dennem begge: Afften- og Morgen-Sange, i Talletd 37, Item Morgen- og Afften-Sucke\n",
      "\n",
      "Digte \n",
      "Digte\n",
      "\n",
      "Digte \n",
      "Digte\n",
      "\n",
      "Digte \n",
      "Digte\n",
      "\n",
      "Udvalgte digte \n",
      "Udvalgte digte\n",
      "\n",
      "Peer Gynt et dramatisk Digt\n",
      "Peer Gynt : et dramatisk Digt\n",
      "\n",
      "Brand et dramatisk Digt\n",
      "Brand : et dramatisk Digt\n",
      "\n",
      "Brand et dramatisk digt\n",
      "Brand : et dramatisk digt\n",
      "\n",
      "Digte \n",
      "Digte\n",
      "\n",
      "Sang ved Studenternes Hjemkomst fra Modet i K\n",
      "Sang ved Studenternes Hjemkomst fra Mödet i Kjöbenhavn : den 22de Juni 1862\n",
      "\n",
      "Brand et dramatisk digt\n",
      "Brand : et dramatisk digt\n",
      "\n",
      "Fanesang \n",
      "Fanesang\n",
      "\n",
      "Digte \n",
      "Digte\n",
      "\n",
      "Sorg og Trost nogle Digte\n",
      "Sorg og Trøst : nogle Digte\n",
      "\n",
      "Nor et Digt\n",
      "Nor : et Digt\n",
      "\n",
      "Jens Zetlitz s samlede Digte \n",
      "Jens Zetlitz's samlede Digte : 1ste Deel\n",
      "\n",
      "Sang foredragen af Bergens Sangforeninger ved\n",
      "Sang : foredragen af Bergens Sangforeninger ved den Koncert, Ole Bull og Sangforeningerne gav til Eidsvoldsmanden Christies Monument den 17de Mai 1859\n",
      "\n",
      "Studenternes sang ved professor C Hansteens b\n",
      "Studenternes sang ved professor C. Hansteens båre i universitetets hal den 21de april 1873\n",
      "\n",
      "Ved Kong Oscars Dod Sang Prolog i Det norske \n",
      "Ved Kong Oscars Død : Sang-Prolog i \"Det norske Theater\" den 24de Juli 1859\n",
      "\n",
      "Sange og stemninger \n",
      "Sange og stemninger\n",
      "\n",
      "Peder Dass s Levnetslob i en Sang tilegnet ha\n",
      "Peder Dass's Levnetsløb : i en Sang, tilegnet hans kjære Menighed : [Min' elskelig' Tilhører' ...]\n",
      "\n",
      "Emilias Kilde et Digt\n",
      "Emilias Kilde : Et Digt\n",
      "\n",
      "Digte forste Ring\n",
      "Digte : Første Ring\n",
      "\n",
      "Menneskesonnen et digt\n",
      "Menneskesønnen : et digt\n",
      "\n",
      "Ildprofeten et episk digt\n",
      "Ildprofeten : et episk digt\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for (title1, title2 ) in zip(check1_df[\"title\"], check2_df[\"title\"]):\n",
    "    print(f\"{title1}\\n{title2}\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Only match on title instead of filename\n",
    "6 books were not match on title, but on author name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "actual_poems = [t for t in df_593[\"title\"] if re.match(pattern, t)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "      <th>filename code</th>\n",
       "      <th>api code</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1830</td>\n",
       "      <td>Guldberg, O. Høegh (Ove)</td>\n",
       "      <td>Psalmebog, eller En Samling af gamle og nye Ps...</td>\n",
       "      <td>2010063003020</td>\n",
       "      <td>2010063003020</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1869</td>\n",
       "      <td>Guldberg, O. Høegh (Ove)</td>\n",
       "      <td>Fra nordiske Digtere : et Album</td>\n",
       "      <td>2006112101006</td>\n",
       "      <td>2006112101006</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1862</td>\n",
       "      <td>Barfod, Frederik</td>\n",
       "      <td>Ingemanns Jordefærd : Taler, Digte og Bladarti...</td>\n",
       "      <td>2009090403002</td>\n",
       "      <td>2009090403002</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1856</td>\n",
       "      <td>Barfod, Frederik</td>\n",
       "      <td>Guds Man : en Psalme-Bog</td>\n",
       "      <td>2006111600006</td>\n",
       "      <td>2006111600006</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1834</td>\n",
       "      <td>Grøndahl, Chr. (Christopher)</td>\n",
       "      <td>Evangelisk-christelig Psalmebog, tilligemed Co...</td>\n",
       "      <td>2008051403001</td>\n",
       "      <td>2008051403001</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>588</th>\n",
       "      <td>1925</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Stene i strømmen : digte</td>\n",
       "      <td>2013070308043</td>\n",
       "      <td>2013070308043</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>589</th>\n",
       "      <td>1947</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Samlede dikt</td>\n",
       "      <td>2012082408040</td>\n",
       "      <td>2012082408040</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>590</th>\n",
       "      <td>1950</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Samlede dikt</td>\n",
       "      <td>2011102708060</td>\n",
       "      <td>2011102708060</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>591</th>\n",
       "      <td>1912</td>\n",
       "      <td>Munthe, Margrethe</td>\n",
       "      <td>Aase fiskerpike : eventyrkomedie med sange til...</td>\n",
       "      <td>2013073008139</td>\n",
       "      <td>2013073008139</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>592</th>\n",
       "      <td>1862</td>\n",
       "      <td>Johnsen, J.</td>\n",
       "      <td>En aandelig Sangbog for angerfulde, bodfærdige...</td>\n",
       "      <td>2010092803011</td>\n",
       "      <td>2010092803011</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>587 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     year                        author  \\\n",
       "0    1830      Guldberg, O. Høegh (Ove)   \n",
       "1    1869      Guldberg, O. Høegh (Ove)   \n",
       "2    1862              Barfod, Frederik   \n",
       "3    1856              Barfod, Frederik   \n",
       "4    1834  Grøndahl, Chr. (Christopher)   \n",
       "..    ...                           ...   \n",
       "588  1925                Grieg, Nordahl   \n",
       "589  1947                Grieg, Nordahl   \n",
       "590  1950                Grieg, Nordahl   \n",
       "591  1912             Munthe, Margrethe   \n",
       "592  1862                   Johnsen, J.   \n",
       "\n",
       "                                                 title  filename code  \\\n",
       "0    Psalmebog, eller En Samling af gamle og nye Ps...  2010063003020   \n",
       "1                      Fra nordiske Digtere : et Album  2006112101006   \n",
       "2    Ingemanns Jordefærd : Taler, Digte og Bladarti...  2009090403002   \n",
       "3                             Guds Man : en Psalme-Bog  2006111600006   \n",
       "4    Evangelisk-christelig Psalmebog, tilligemed Co...  2008051403001   \n",
       "..                                                 ...            ...   \n",
       "588                           Stene i strømmen : digte  2013070308043   \n",
       "589                                       Samlede dikt  2012082408040   \n",
       "590                                       Samlede dikt  2011102708060   \n",
       "591  Aase fiskerpike : eventyrkomedie med sange til...  2013073008139   \n",
       "592  En aandelig Sangbog for angerfulde, bodfærdige...  2010092803011   \n",
       "\n",
       "          api code lang  \n",
       "0    2010063003020  dan  \n",
       "1    2006112101006  dan  \n",
       "2    2009090403002  dan  \n",
       "3    2006111600006  nob  \n",
       "4    2008051403001  dan  \n",
       "..             ...  ...  \n",
       "588  2013070308043  nob  \n",
       "589  2012082408040  nob  \n",
       "590  2011102708060  nob  \n",
       "591  2013073008139  nob  \n",
       "592  2010092803011  nob  \n",
       "\n",
       "[587 rows x 6 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ = df_593.loc[df_593[\"title\"].isin(actual_poems)]\n",
    "df_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Delete unwanted files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "      <th>filename code</th>\n",
       "      <th>api code</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1860</td>\n",
       "      <td>Ahnfelt, Oscar</td>\n",
       "      <td>Melodierna i sifferskrift till \"Andeliga sånger\"</td>\n",
       "      <td>2009101303013</td>\n",
       "      <td>2009101303013</td>\n",
       "      <td>swe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>1786</td>\n",
       "      <td>Bach, Johann Sebastian</td>\n",
       "      <td>Vierstimmige alte und neue Choralgesänge mit P...</td>\n",
       "      <td>2012012424002</td>\n",
       "      <td>2012012424002</td>\n",
       "      <td>ger</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>1814</td>\n",
       "      <td>Sohm, Peter.</td>\n",
       "      <td>Sång på Årsdagen af Bataillen vid Leipzig, den...</td>\n",
       "      <td>2013062726004</td>\n",
       "      <td>2013062726004</td>\n",
       "      <td>swe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>256</th>\n",
       "      <td>1861</td>\n",
       "      <td>Runeberg, Johan Ludvig</td>\n",
       "      <td>Julqvällen : tre sånger</td>\n",
       "      <td>2009090203012</td>\n",
       "      <td>2009090203012</td>\n",
       "      <td>swe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>512</th>\n",
       "      <td>1966</td>\n",
       "      <td>Jøsang, Ottar</td>\n",
       "      <td>Geologiske og petrografiske undersøkelser i Mo...</td>\n",
       "      <td>2007042001008</td>\n",
       "      <td>2007042001008</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548</th>\n",
       "      <td>1814</td>\n",
       "      <td>Valerius, Johan David</td>\n",
       "      <td>Förenings-sång</td>\n",
       "      <td>2011112224037</td>\n",
       "      <td>2011112224037</td>\n",
       "      <td>mul</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     year                  author  \\\n",
       "17   1860          Ahnfelt, Oscar   \n",
       "37   1786  Bach, Johann Sebastian   \n",
       "41   1814            Sohm, Peter.   \n",
       "256  1861  Runeberg, Johan Ludvig   \n",
       "512  1966           Jøsang, Ottar   \n",
       "548  1814   Valerius, Johan David   \n",
       "\n",
       "                                                 title  filename code  \\\n",
       "17    Melodierna i sifferskrift till \"Andeliga sånger\"  2009101303013   \n",
       "37   Vierstimmige alte und neue Choralgesänge mit P...  2012012424002   \n",
       "41   Sång på Årsdagen af Bataillen vid Leipzig, den...  2013062726004   \n",
       "256                            Julqvällen : tre sånger  2009090203012   \n",
       "512  Geologiske og petrografiske undersøkelser i Mo...  2007042001008   \n",
       "548                                     Förenings-sång  2011112224037   \n",
       "\n",
       "          api code lang  \n",
       "17   2009101303013  swe  \n",
       "37   2012012424002  ger  \n",
       "41   2013062726004  swe  \n",
       "256  2009090203012  swe  \n",
       "512  2007042001008  nob  \n",
       "548  2011112224037  mul  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "files_to_delete = df_593.loc[~df_593[\"title\"].isin(actual_poems)]\n",
    "files_to_delete"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Remove poems that are not Norwegian by filename"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Of the 593 poems selected on filename, 452 of them have \"nob\", \"nno\" or \"nor\" in the filename. \n",
    "That means 141 are in different languages. However several of these are danish in the dataframe!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "452"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "langs = [(e.name, e.name.split(\"-\")[2]) for e in poems]\n",
    "norw = [(name, lang) for name, lang in langs if lang in (\"nob\", \"nno\", \"nor\")]\n",
    "len(norw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "      <th>filename code</th>\n",
       "      <th>api code</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1830</td>\n",
       "      <td>Guldberg, O. Høegh (Ove)</td>\n",
       "      <td>Psalmebog, eller En Samling af gamle og nye Ps...</td>\n",
       "      <td>2010063003020</td>\n",
       "      <td>2010063003020</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1856</td>\n",
       "      <td>Barfod, Frederik</td>\n",
       "      <td>Guds Man : en Psalme-Bog</td>\n",
       "      <td>2006111600006</td>\n",
       "      <td>2006111600006</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1834</td>\n",
       "      <td>Grøndahl, Chr. (Christopher)</td>\n",
       "      <td>Evangelisk-christelig Psalmebog, tilligemed Co...</td>\n",
       "      <td>2008051403001</td>\n",
       "      <td>2008051403001</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1852</td>\n",
       "      <td>Fearnley, Henry Emil</td>\n",
       "      <td>Smaadigte til Udenadslæsning : til Brug ved de...</td>\n",
       "      <td>2010031003059</td>\n",
       "      <td>2010031003059</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1852</td>\n",
       "      <td>Hauge, A. Andreas</td>\n",
       "      <td>100 Missions-Psalmer ; med et Tillæg, indehold...</td>\n",
       "      <td>2006111400042</td>\n",
       "      <td>2006111400042</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>588</th>\n",
       "      <td>1925</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Stene i strømmen : digte</td>\n",
       "      <td>2013070308043</td>\n",
       "      <td>2013070308043</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>589</th>\n",
       "      <td>1947</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Samlede dikt</td>\n",
       "      <td>2012082408040</td>\n",
       "      <td>2012082408040</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>590</th>\n",
       "      <td>1950</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Samlede dikt</td>\n",
       "      <td>2011102708060</td>\n",
       "      <td>2011102708060</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>591</th>\n",
       "      <td>1912</td>\n",
       "      <td>Munthe, Margrethe</td>\n",
       "      <td>Aase fiskerpike : eventyrkomedie med sange til...</td>\n",
       "      <td>2013073008139</td>\n",
       "      <td>2013073008139</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>592</th>\n",
       "      <td>1862</td>\n",
       "      <td>Johnsen, J.</td>\n",
       "      <td>En aandelig Sangbog for angerfulde, bodfærdige...</td>\n",
       "      <td>2010092803011</td>\n",
       "      <td>2010092803011</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>452 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     year                        author  \\\n",
       "0    1830      Guldberg, O. Høegh (Ove)   \n",
       "3    1856              Barfod, Frederik   \n",
       "4    1834  Grøndahl, Chr. (Christopher)   \n",
       "5    1852          Fearnley, Henry Emil   \n",
       "6    1852             Hauge, A. Andreas   \n",
       "..    ...                           ...   \n",
       "588  1925                Grieg, Nordahl   \n",
       "589  1947                Grieg, Nordahl   \n",
       "590  1950                Grieg, Nordahl   \n",
       "591  1912             Munthe, Margrethe   \n",
       "592  1862                   Johnsen, J.   \n",
       "\n",
       "                                                 title  filename code  \\\n",
       "0    Psalmebog, eller En Samling af gamle og nye Ps...  2010063003020   \n",
       "3                             Guds Man : en Psalme-Bog  2006111600006   \n",
       "4    Evangelisk-christelig Psalmebog, tilligemed Co...  2008051403001   \n",
       "5    Smaadigte til Udenadslæsning : til Brug ved de...  2010031003059   \n",
       "6    100 Missions-Psalmer ; med et Tillæg, indehold...  2006111400042   \n",
       "..                                                 ...            ...   \n",
       "588                           Stene i strømmen : digte  2013070308043   \n",
       "589                                       Samlede dikt  2012082408040   \n",
       "590                                       Samlede dikt  2011102708060   \n",
       "591  Aase fiskerpike : eventyrkomedie med sange til...  2013073008139   \n",
       "592  En aandelig Sangbog for angerfulde, bodfærdige...  2010092803011   \n",
       "\n",
       "          api code lang  \n",
       "0    2010063003020  dan  \n",
       "3    2006111600006  nob  \n",
       "4    2008051403001  dan  \n",
       "5    2010031003059  nob  \n",
       "6    2006111400042  nob  \n",
       "..             ...  ...  \n",
       "588  2013070308043  nob  \n",
       "589  2012082408040  nob  \n",
       "590  2011102708060  nob  \n",
       "591  2013073008139  nob  \n",
       "592  2010092803011  nob  \n",
       "\n",
       "[452 rows x 6 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "norw_codes = [get_code(name) for name, lang in norw]\n",
    "norw_df = df_593.loc[df_593[\"filename code\"].isin(norw_codes)]\n",
    "norw_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If we also remove the ones that dont have a title match we get 451 books"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "      <th>filename code</th>\n",
       "      <th>api code</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1830</td>\n",
       "      <td>Guldberg, O. Høegh (Ove)</td>\n",
       "      <td>Psalmebog, eller En Samling af gamle og nye Ps...</td>\n",
       "      <td>2010063003020</td>\n",
       "      <td>2010063003020</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1856</td>\n",
       "      <td>Barfod, Frederik</td>\n",
       "      <td>Guds Man : en Psalme-Bog</td>\n",
       "      <td>2006111600006</td>\n",
       "      <td>2006111600006</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1834</td>\n",
       "      <td>Grøndahl, Chr. (Christopher)</td>\n",
       "      <td>Evangelisk-christelig Psalmebog, tilligemed Co...</td>\n",
       "      <td>2008051403001</td>\n",
       "      <td>2008051403001</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1852</td>\n",
       "      <td>Fearnley, Henry Emil</td>\n",
       "      <td>Smaadigte til Udenadslæsning : til Brug ved de...</td>\n",
       "      <td>2010031003059</td>\n",
       "      <td>2010031003059</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1852</td>\n",
       "      <td>Hauge, A. Andreas</td>\n",
       "      <td>100 Missions-Psalmer ; med et Tillæg, indehold...</td>\n",
       "      <td>2006111400042</td>\n",
       "      <td>2006111400042</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>588</th>\n",
       "      <td>1925</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Stene i strømmen : digte</td>\n",
       "      <td>2013070308043</td>\n",
       "      <td>2013070308043</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>589</th>\n",
       "      <td>1947</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Samlede dikt</td>\n",
       "      <td>2012082408040</td>\n",
       "      <td>2012082408040</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>590</th>\n",
       "      <td>1950</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Samlede dikt</td>\n",
       "      <td>2011102708060</td>\n",
       "      <td>2011102708060</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>591</th>\n",
       "      <td>1912</td>\n",
       "      <td>Munthe, Margrethe</td>\n",
       "      <td>Aase fiskerpike : eventyrkomedie med sange til...</td>\n",
       "      <td>2013073008139</td>\n",
       "      <td>2013073008139</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>592</th>\n",
       "      <td>1862</td>\n",
       "      <td>Johnsen, J.</td>\n",
       "      <td>En aandelig Sangbog for angerfulde, bodfærdige...</td>\n",
       "      <td>2010092803011</td>\n",
       "      <td>2010092803011</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>451 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     year                        author  \\\n",
       "0    1830      Guldberg, O. Høegh (Ove)   \n",
       "3    1856              Barfod, Frederik   \n",
       "4    1834  Grøndahl, Chr. (Christopher)   \n",
       "5    1852          Fearnley, Henry Emil   \n",
       "6    1852             Hauge, A. Andreas   \n",
       "..    ...                           ...   \n",
       "588  1925                Grieg, Nordahl   \n",
       "589  1947                Grieg, Nordahl   \n",
       "590  1950                Grieg, Nordahl   \n",
       "591  1912             Munthe, Margrethe   \n",
       "592  1862                   Johnsen, J.   \n",
       "\n",
       "                                                 title  filename code  \\\n",
       "0    Psalmebog, eller En Samling af gamle og nye Ps...  2010063003020   \n",
       "3                             Guds Man : en Psalme-Bog  2006111600006   \n",
       "4    Evangelisk-christelig Psalmebog, tilligemed Co...  2008051403001   \n",
       "5    Smaadigte til Udenadslæsning : til Brug ved de...  2010031003059   \n",
       "6    100 Missions-Psalmer ; med et Tillæg, indehold...  2006111400042   \n",
       "..                                                 ...            ...   \n",
       "588                           Stene i strømmen : digte  2013070308043   \n",
       "589                                       Samlede dikt  2012082408040   \n",
       "590                                       Samlede dikt  2011102708060   \n",
       "591  Aase fiskerpike : eventyrkomedie med sange til...  2013073008139   \n",
       "592  En aandelig Sangbog for angerfulde, bodfærdige...  2010092803011   \n",
       "\n",
       "          api code lang  \n",
       "0    2010063003020  dan  \n",
       "3    2006111600006  nob  \n",
       "4    2008051403001  dan  \n",
       "5    2010031003059  nob  \n",
       "6    2006111400042  nob  \n",
       "..             ...  ...  \n",
       "588  2013070308043  nob  \n",
       "589  2012082408040  nob  \n",
       "590  2011102708060  nob  \n",
       "591  2013073008139  nob  \n",
       "592  2010092803011  nob  \n",
       "\n",
       "[451 rows x 6 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "norw_df = df_.loc[df_[\"filename code\"].isin(norw_codes)]\n",
    "norw_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Removing all books that are not listed as nob, nno, nor in metadata info. These are 46 books"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "46\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "      <th>filename code</th>\n",
       "      <th>api code</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1830</td>\n",
       "      <td>Guldberg, O. Høegh (Ove)</td>\n",
       "      <td>Psalmebog, eller En Samling af gamle og nye Ps...</td>\n",
       "      <td>2010063003020</td>\n",
       "      <td>2010063003020</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1834</td>\n",
       "      <td>Grøndahl, Chr. (Christopher)</td>\n",
       "      <td>Evangelisk-christelig Psalmebog, tilligemed Co...</td>\n",
       "      <td>2008051403001</td>\n",
       "      <td>2008051403001</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1903</td>\n",
       "      <td>Collin, Chr.</td>\n",
       "      <td>Engelske digte og sange</td>\n",
       "      <td>2012052505023</td>\n",
       "      <td>2012052505023</td>\n",
       "      <td>mul</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1922</td>\n",
       "      <td>Hægstad, Marius</td>\n",
       "      <td>Norsk diktning efter 1814</td>\n",
       "      <td>2006120101048</td>\n",
       "      <td>2006120101048</td>\n",
       "      <td>mul</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1843</td>\n",
       "      <td>Cappelen, Jørgen Wright</td>\n",
       "      <td>Evangelisk-christelig Psalmebog, tilligemed Co...</td>\n",
       "      <td>2008102303005</td>\n",
       "      <td>2008102303005</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1834</td>\n",
       "      <td>Kingo, Thomas</td>\n",
       "      <td>Psalme-Skat : Et Udvalg af gamle Psalmer, uddr...</td>\n",
       "      <td>2008042303004</td>\n",
       "      <td>2021120826007</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>1796</td>\n",
       "      <td>Madsen, Johan</td>\n",
       "      <td>Zions Chor-Sange mod Verdens Synge-Chor. Der e...</td>\n",
       "      <td>2009031610002</td>\n",
       "      <td>2009031610002</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>1848</td>\n",
       "      <td>Cappelen, Jørgen Wright</td>\n",
       "      <td>Evangelisk-christelig Psalmebog, tilligemed Co...</td>\n",
       "      <td>2008102303002</td>\n",
       "      <td>2015071427011</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>1799</td>\n",
       "      <td>Brorson, Hans Adolph</td>\n",
       "      <td>De Sande Christnes udvalgte Psalmebog, uddrage...</td>\n",
       "      <td>2009030610001</td>\n",
       "      <td>2009030610001</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>1844</td>\n",
       "      <td>Cappelen, Jørgen Wright</td>\n",
       "      <td>Evangelisk-christelig Psalmebog, tilligemed Co...</td>\n",
       "      <td>2008102303001</td>\n",
       "      <td>2008102303001</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>1845</td>\n",
       "      <td>Dahl, Chr. (Christopher)</td>\n",
       "      <td>Det Svundne Aar : Et Digt</td>\n",
       "      <td>2011081913004</td>\n",
       "      <td>2011081913004</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>1850</td>\n",
       "      <td>Koren, Vilhelm</td>\n",
       "      <td>Digte til Udenadlæsning, til Brug for Skolerne...</td>\n",
       "      <td>2008071700087</td>\n",
       "      <td>2008071700087</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>1883</td>\n",
       "      <td>Lindeman, Ludv. M. (Ludvig Mathias)</td>\n",
       "      <td>Koralbog : indeholdende de i Landstads Salmebo...</td>\n",
       "      <td>2012011012001</td>\n",
       "      <td>2012011012001</td>\n",
       "      <td>mul</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>110</th>\n",
       "      <td>1848</td>\n",
       "      <td>Heyerdahl, Halvor</td>\n",
       "      <td>Historiske Sange fra Norges Fortid</td>\n",
       "      <td>2010020103018</td>\n",
       "      <td>2010020103018</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>1845</td>\n",
       "      <td>Welhaven, J.S. (Johan Sebastian)</td>\n",
       "      <td>Nyere Digte</td>\n",
       "      <td>2010100608041</td>\n",
       "      <td>2010100608041</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128</th>\n",
       "      <td>1807</td>\n",
       "      <td>Bonnevie, Andreas</td>\n",
       "      <td>Krigs-Sang for Kronprinsens Livcorps</td>\n",
       "      <td>2012071113009</td>\n",
       "      <td>2012071113009</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154</th>\n",
       "      <td>1848</td>\n",
       "      <td>Winsnes, Hanna</td>\n",
       "      <td>Smaa-Digte</td>\n",
       "      <td>2009111912011</td>\n",
       "      <td>2009111912011</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>165</th>\n",
       "      <td>1816</td>\n",
       "      <td>Olsen, Andreas Schram</td>\n",
       "      <td>Digte : Første Deel</td>\n",
       "      <td>2011050308007</td>\n",
       "      <td>2011050308007</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>196</th>\n",
       "      <td>1801</td>\n",
       "      <td>Schmidt, Frederik</td>\n",
       "      <td>Sang for de danske og norske Søekrigere</td>\n",
       "      <td>2012071213004</td>\n",
       "      <td>2012071213004</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201</th>\n",
       "      <td>1898</td>\n",
       "      <td>Monrad, M.J.</td>\n",
       "      <td>Efterladte Digte</td>\n",
       "      <td>2008042812014</td>\n",
       "      <td>2008042812014</td>\n",
       "      <td>mul</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>241</th>\n",
       "      <td>1814</td>\n",
       "      <td>Wichne, Arne Erichsen</td>\n",
       "      <td>Normands-Sang</td>\n",
       "      <td>2008100110004</td>\n",
       "      <td>2008100110004</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>242</th>\n",
       "      <td>1814</td>\n",
       "      <td>Pavels, Claus</td>\n",
       "      <td>Taler og Sange i Aggershuus Slotskirke paa den...</td>\n",
       "      <td>2012030212009</td>\n",
       "      <td>2013032526002</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>243</th>\n",
       "      <td>1801</td>\n",
       "      <td>Pavels, Claus</td>\n",
       "      <td>Borgersang</td>\n",
       "      <td>2012071213001</td>\n",
       "      <td>2012071213001</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246</th>\n",
       "      <td>1845</td>\n",
       "      <td>Wessel, Johan Herman</td>\n",
       "      <td>Samlede digte af Johan Herman Wessel. Udgivne ...</td>\n",
       "      <td>2012070608276</td>\n",
       "      <td>2012070608276</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>247</th>\n",
       "      <td>1801</td>\n",
       "      <td>Heber, Carl Vilhelm</td>\n",
       "      <td>Sang for danske Søemænd : af C. Heber, resider...</td>\n",
       "      <td>2012071113010</td>\n",
       "      <td>2012071113010</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>306</th>\n",
       "      <td>1816</td>\n",
       "      <td>Hansen, Maurits</td>\n",
       "      <td>Digtninger</td>\n",
       "      <td>2011050308010</td>\n",
       "      <td>2011050308010</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>307</th>\n",
       "      <td>1825</td>\n",
       "      <td>Hansen, Maurits</td>\n",
       "      <td>Digtninger : Anden Deel</td>\n",
       "      <td>2007082703001</td>\n",
       "      <td>2007082703001</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>316</th>\n",
       "      <td>1891</td>\n",
       "      <td>Hougen, Engebret</td>\n",
       "      <td>Digte</td>\n",
       "      <td>2006112401018</td>\n",
       "      <td>2006112401018</td>\n",
       "      <td>mul</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>319</th>\n",
       "      <td>1847</td>\n",
       "      <td>Kokk, Bernhard</td>\n",
       "      <td>Digte</td>\n",
       "      <td>2007052210005</td>\n",
       "      <td>2007052210005</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>329</th>\n",
       "      <td>1848</td>\n",
       "      <td>Bion Smyrnaeus</td>\n",
       "      <td>Idylliske Digte</td>\n",
       "      <td>2011050308009</td>\n",
       "      <td>2011050308009</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>346</th>\n",
       "      <td>1835</td>\n",
       "      <td>Foss, Herman</td>\n",
       "      <td>Tidsnornerne : Et Leilighedsdigt</td>\n",
       "      <td>2009061912002</td>\n",
       "      <td>2009061912002</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>358</th>\n",
       "      <td>1915</td>\n",
       "      <td>Baumann, Julius Berg</td>\n",
       "      <td>Fra vidderne : nye digte</td>\n",
       "      <td>2006120500055</td>\n",
       "      <td>2006120500055</td>\n",
       "      <td>mul</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>376</th>\n",
       "      <td>1795</td>\n",
       "      <td>Zetlitz, Jens</td>\n",
       "      <td>Sange for den norske Bonde-Stand</td>\n",
       "      <td>2012012813002</td>\n",
       "      <td>2012012813002</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>378</th>\n",
       "      <td>1795</td>\n",
       "      <td>Zetlitz, Jens</td>\n",
       "      <td>Psalmer</td>\n",
       "      <td>2009032710003</td>\n",
       "      <td>2009032710003</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>423</th>\n",
       "      <td>1849</td>\n",
       "      <td>Andresen, Peter Rasmussen</td>\n",
       "      <td>Bibelske Krønike-Riim, eller historiske Psalme...</td>\n",
       "      <td>2010042703061</td>\n",
       "      <td>2010042703061</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>431</th>\n",
       "      <td>1796</td>\n",
       "      <td>Brun, Johan Nordahl</td>\n",
       "      <td>Jonathan : Et Digt i ti Sange</td>\n",
       "      <td>2009040112004</td>\n",
       "      <td>2009040112004</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>439</th>\n",
       "      <td>1956</td>\n",
       "      <td>Dass, Petter</td>\n",
       "      <td>Herrens bøn, Fader vor : forfattet udi ni sange</td>\n",
       "      <td>2011121308005</td>\n",
       "      <td>2011121308005</td>\n",
       "      <td>mul</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>442</th>\n",
       "      <td>1736</td>\n",
       "      <td>Dass, Petter</td>\n",
       "      <td>En skiøn Aandelig Psalme : uddragen af Det syv...</td>\n",
       "      <td>2009072713011</td>\n",
       "      <td>2009072713011</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>461</th>\n",
       "      <td>1897</td>\n",
       "      <td>Jaastad, Lars</td>\n",
       "      <td>Til Fjells : Dikt</td>\n",
       "      <td>2008040200055</td>\n",
       "      <td>2008040200055</td>\n",
       "      <td>mul</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>481</th>\n",
       "      <td>1851</td>\n",
       "      <td>Harboe, Ludvig</td>\n",
       "      <td>Psalmebog, eller En Samling af gamle og nye Ps...</td>\n",
       "      <td>2008091503018</td>\n",
       "      <td>2008091503018</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>498</th>\n",
       "      <td>1897</td>\n",
       "      <td>Lie, John</td>\n",
       "      <td>Aarolilja : Dikt</td>\n",
       "      <td>2006112300054</td>\n",
       "      <td>2006112300054</td>\n",
       "      <td>mul</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>510</th>\n",
       "      <td>1801</td>\n",
       "      <td>Wulfsberg, Niels</td>\n",
       "      <td>Sang for Kronprindsens Livcorps</td>\n",
       "      <td>2012071213005</td>\n",
       "      <td>2012071213005</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>522</th>\n",
       "      <td>1845</td>\n",
       "      <td>Wergeland, Henrik</td>\n",
       "      <td>Mennesket : Et Digt</td>\n",
       "      <td>2009091403003</td>\n",
       "      <td>2009091403003</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>523</th>\n",
       "      <td>1846</td>\n",
       "      <td>Wergeland, Henrik</td>\n",
       "      <td>Udvalgte lyriske Digte</td>\n",
       "      <td>2012070608265</td>\n",
       "      <td>2012070608265</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>526</th>\n",
       "      <td>1829</td>\n",
       "      <td>Wergeland, Henrik</td>\n",
       "      <td>Digte : Første Ring</td>\n",
       "      <td>2013040824027</td>\n",
       "      <td>2015111826003</td>\n",
       "      <td>dan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>550</th>\n",
       "      <td>1864</td>\n",
       "      <td>Vinje, A.O. (Aasmund Olavsson)</td>\n",
       "      <td>Diktsamling</td>\n",
       "      <td>2009071303002</td>\n",
       "      <td>2009071303002</td>\n",
       "      <td>mul</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     year                               author  \\\n",
       "0    1830             Guldberg, O. Høegh (Ove)   \n",
       "4    1834         Grøndahl, Chr. (Christopher)   \n",
       "8    1903                         Collin, Chr.   \n",
       "9    1922                      Hægstad, Marius   \n",
       "12   1843              Cappelen, Jørgen Wright   \n",
       "14   1834                        Kingo, Thomas   \n",
       "34   1796                        Madsen, Johan   \n",
       "49   1848              Cappelen, Jørgen Wright   \n",
       "50   1799                 Brorson, Hans Adolph   \n",
       "55   1844              Cappelen, Jørgen Wright   \n",
       "71   1845             Dahl, Chr. (Christopher)   \n",
       "82   1850                       Koren, Vilhelm   \n",
       "94   1883  Lindeman, Ludv. M. (Ludvig Mathias)   \n",
       "110  1848                    Heyerdahl, Halvor   \n",
       "117  1845     Welhaven, J.S. (Johan Sebastian)   \n",
       "128  1807                    Bonnevie, Andreas   \n",
       "154  1848                       Winsnes, Hanna   \n",
       "165  1816                Olsen, Andreas Schram   \n",
       "196  1801                    Schmidt, Frederik   \n",
       "201  1898                         Monrad, M.J.   \n",
       "241  1814                Wichne, Arne Erichsen   \n",
       "242  1814                        Pavels, Claus   \n",
       "243  1801                        Pavels, Claus   \n",
       "246  1845                 Wessel, Johan Herman   \n",
       "247  1801                  Heber, Carl Vilhelm   \n",
       "306  1816                      Hansen, Maurits   \n",
       "307  1825                      Hansen, Maurits   \n",
       "316  1891                     Hougen, Engebret   \n",
       "319  1847                       Kokk, Bernhard   \n",
       "329  1848                       Bion Smyrnaeus   \n",
       "346  1835                         Foss, Herman   \n",
       "358  1915                 Baumann, Julius Berg   \n",
       "376  1795                        Zetlitz, Jens   \n",
       "378  1795                        Zetlitz, Jens   \n",
       "423  1849            Andresen, Peter Rasmussen   \n",
       "431  1796                  Brun, Johan Nordahl   \n",
       "439  1956                         Dass, Petter   \n",
       "442  1736                         Dass, Petter   \n",
       "461  1897                        Jaastad, Lars   \n",
       "481  1851                       Harboe, Ludvig   \n",
       "498  1897                            Lie, John   \n",
       "510  1801                     Wulfsberg, Niels   \n",
       "522  1845                    Wergeland, Henrik   \n",
       "523  1846                    Wergeland, Henrik   \n",
       "526  1829                    Wergeland, Henrik   \n",
       "550  1864       Vinje, A.O. (Aasmund Olavsson)   \n",
       "\n",
       "                                                 title  filename code  \\\n",
       "0    Psalmebog, eller En Samling af gamle og nye Ps...  2010063003020   \n",
       "4    Evangelisk-christelig Psalmebog, tilligemed Co...  2008051403001   \n",
       "8                              Engelske digte og sange  2012052505023   \n",
       "9                            Norsk diktning efter 1814  2006120101048   \n",
       "12   Evangelisk-christelig Psalmebog, tilligemed Co...  2008102303005   \n",
       "14   Psalme-Skat : Et Udvalg af gamle Psalmer, uddr...  2008042303004   \n",
       "34   Zions Chor-Sange mod Verdens Synge-Chor. Der e...  2009031610002   \n",
       "49   Evangelisk-christelig Psalmebog, tilligemed Co...  2008102303002   \n",
       "50   De Sande Christnes udvalgte Psalmebog, uddrage...  2009030610001   \n",
       "55   Evangelisk-christelig Psalmebog, tilligemed Co...  2008102303001   \n",
       "71                           Det Svundne Aar : Et Digt  2011081913004   \n",
       "82   Digte til Udenadlæsning, til Brug for Skolerne...  2008071700087   \n",
       "94   Koralbog : indeholdende de i Landstads Salmebo...  2012011012001   \n",
       "110                 Historiske Sange fra Norges Fortid  2010020103018   \n",
       "117                                        Nyere Digte  2010100608041   \n",
       "128               Krigs-Sang for Kronprinsens Livcorps  2012071113009   \n",
       "154                                         Smaa-Digte  2009111912011   \n",
       "165                                Digte : Første Deel  2011050308007   \n",
       "196            Sang for de danske og norske Søekrigere  2012071213004   \n",
       "201                                   Efterladte Digte  2008042812014   \n",
       "241                                      Normands-Sang  2008100110004   \n",
       "242  Taler og Sange i Aggershuus Slotskirke paa den...  2012030212009   \n",
       "243                                         Borgersang  2012071213001   \n",
       "246  Samlede digte af Johan Herman Wessel. Udgivne ...  2012070608276   \n",
       "247  Sang for danske Søemænd : af C. Heber, resider...  2012071113010   \n",
       "306                                         Digtninger  2011050308010   \n",
       "307                            Digtninger : Anden Deel  2007082703001   \n",
       "316                                              Digte  2006112401018   \n",
       "319                                              Digte  2007052210005   \n",
       "329                                    Idylliske Digte  2011050308009   \n",
       "346                   Tidsnornerne : Et Leilighedsdigt  2009061912002   \n",
       "358                           Fra vidderne : nye digte  2006120500055   \n",
       "376                   Sange for den norske Bonde-Stand  2012012813002   \n",
       "378                                            Psalmer  2009032710003   \n",
       "423  Bibelske Krønike-Riim, eller historiske Psalme...  2010042703061   \n",
       "431                      Jonathan : Et Digt i ti Sange  2009040112004   \n",
       "439    Herrens bøn, Fader vor : forfattet udi ni sange  2011121308005   \n",
       "442  En skiøn Aandelig Psalme : uddragen af Det syv...  2009072713011   \n",
       "461                                  Til Fjells : Dikt  2008040200055   \n",
       "481  Psalmebog, eller En Samling af gamle og nye Ps...  2008091503018   \n",
       "498                                   Aarolilja : Dikt  2006112300054   \n",
       "510                    Sang for Kronprindsens Livcorps  2012071213005   \n",
       "522                                Mennesket : Et Digt  2009091403003   \n",
       "523                             Udvalgte lyriske Digte  2012070608265   \n",
       "526                                Digte : Første Ring  2013040824027   \n",
       "550                                        Diktsamling  2009071303002   \n",
       "\n",
       "          api code lang  \n",
       "0    2010063003020  dan  \n",
       "4    2008051403001  dan  \n",
       "8    2012052505023  mul  \n",
       "9    2006120101048  mul  \n",
       "12   2008102303005  dan  \n",
       "14   2021120826007  dan  \n",
       "34   2009031610002  dan  \n",
       "49   2015071427011  dan  \n",
       "50   2009030610001  dan  \n",
       "55   2008102303001  dan  \n",
       "71   2011081913004  dan  \n",
       "82   2008071700087  dan  \n",
       "94   2012011012001  mul  \n",
       "110  2010020103018  dan  \n",
       "117  2010100608041  dan  \n",
       "128  2012071113009  dan  \n",
       "154  2009111912011  dan  \n",
       "165  2011050308007  dan  \n",
       "196  2012071213004  dan  \n",
       "201  2008042812014  mul  \n",
       "241  2008100110004  dan  \n",
       "242  2013032526002  dan  \n",
       "243  2012071213001  dan  \n",
       "246  2012070608276  dan  \n",
       "247  2012071113010  dan  \n",
       "306  2011050308010  dan  \n",
       "307  2007082703001  dan  \n",
       "316  2006112401018  mul  \n",
       "319  2007052210005  dan  \n",
       "329  2011050308009  dan  \n",
       "346  2009061912002  dan  \n",
       "358  2006120500055  mul  \n",
       "376  2012012813002  dan  \n",
       "378  2009032710003  dan  \n",
       "423  2010042703061  dan  \n",
       "431  2009040112004  dan  \n",
       "439  2011121308005  mul  \n",
       "442  2009072713011  dan  \n",
       "461  2008040200055  mul  \n",
       "481  2008091503018  dan  \n",
       "498  2006112300054  mul  \n",
       "510  2012071213005  dan  \n",
       "522  2009091403003  dan  \n",
       "523  2012070608265  dan  \n",
       "526  2015111826003  dan  \n",
       "550  2009071303002  mul  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "not_norw = norw_df.loc[~norw_df[\"lang\"].isin((\"nob\", \"nno\", \"nor\"))]\n",
    "print(len(not_norw))\n",
    "not_norw"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the end, selecting by filename first, and then by language, yields 405 books"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "      <th>filename code</th>\n",
       "      <th>api code</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1856</td>\n",
       "      <td>Barfod, Frederik</td>\n",
       "      <td>Guds Man : en Psalme-Bog</td>\n",
       "      <td>2006111600006</td>\n",
       "      <td>2006111600006</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1852</td>\n",
       "      <td>Fearnley, Henry Emil</td>\n",
       "      <td>Smaadigte til Udenadslæsning : til Brug ved de...</td>\n",
       "      <td>2010031003059</td>\n",
       "      <td>2010031003059</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1852</td>\n",
       "      <td>Hauge, A. Andreas</td>\n",
       "      <td>100 Missions-Psalmer ; med et Tillæg, indehold...</td>\n",
       "      <td>2006111400042</td>\n",
       "      <td>2006111400042</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1863</td>\n",
       "      <td>Hægstad, Marius</td>\n",
       "      <td>Evangelisk-christelig Psalmebog : tilligemed C...</td>\n",
       "      <td>2008121003001</td>\n",
       "      <td>2008121003001</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1898</td>\n",
       "      <td>Skard, Matias</td>\n",
       "      <td>Norsk Digtning efter 1814 : et Udvalg til Brug...</td>\n",
       "      <td>2010081103001</td>\n",
       "      <td>2010081103001</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>588</th>\n",
       "      <td>1925</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Stene i strømmen : digte</td>\n",
       "      <td>2013070308043</td>\n",
       "      <td>2013070308043</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>589</th>\n",
       "      <td>1947</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Samlede dikt</td>\n",
       "      <td>2012082408040</td>\n",
       "      <td>2012082408040</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>590</th>\n",
       "      <td>1950</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Samlede dikt</td>\n",
       "      <td>2011102708060</td>\n",
       "      <td>2011102708060</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>591</th>\n",
       "      <td>1912</td>\n",
       "      <td>Munthe, Margrethe</td>\n",
       "      <td>Aase fiskerpike : eventyrkomedie med sange til...</td>\n",
       "      <td>2013073008139</td>\n",
       "      <td>2013073008139</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>592</th>\n",
       "      <td>1862</td>\n",
       "      <td>Johnsen, J.</td>\n",
       "      <td>En aandelig Sangbog for angerfulde, bodfærdige...</td>\n",
       "      <td>2010092803011</td>\n",
       "      <td>2010092803011</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>405 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     year                author  \\\n",
       "3    1856      Barfod, Frederik   \n",
       "5    1852  Fearnley, Henry Emil   \n",
       "6    1852     Hauge, A. Andreas   \n",
       "10   1863       Hægstad, Marius   \n",
       "13   1898         Skard, Matias   \n",
       "..    ...                   ...   \n",
       "588  1925        Grieg, Nordahl   \n",
       "589  1947        Grieg, Nordahl   \n",
       "590  1950        Grieg, Nordahl   \n",
       "591  1912     Munthe, Margrethe   \n",
       "592  1862           Johnsen, J.   \n",
       "\n",
       "                                                 title  filename code  \\\n",
       "3                             Guds Man : en Psalme-Bog  2006111600006   \n",
       "5    Smaadigte til Udenadslæsning : til Brug ved de...  2010031003059   \n",
       "6    100 Missions-Psalmer ; med et Tillæg, indehold...  2006111400042   \n",
       "10   Evangelisk-christelig Psalmebog : tilligemed C...  2008121003001   \n",
       "13   Norsk Digtning efter 1814 : et Udvalg til Brug...  2010081103001   \n",
       "..                                                 ...            ...   \n",
       "588                           Stene i strømmen : digte  2013070308043   \n",
       "589                                       Samlede dikt  2012082408040   \n",
       "590                                       Samlede dikt  2011102708060   \n",
       "591  Aase fiskerpike : eventyrkomedie med sange til...  2013073008139   \n",
       "592  En aandelig Sangbog for angerfulde, bodfærdige...  2010092803011   \n",
       "\n",
       "          api code lang  \n",
       "3    2006111600006  nob  \n",
       "5    2010031003059  nob  \n",
       "6    2006111400042  nob  \n",
       "10   2008121003001  nob  \n",
       "13   2010081103001  nob  \n",
       "..             ...  ...  \n",
       "588  2013070308043  nob  \n",
       "589  2012082408040  nob  \n",
       "590  2011102708060  nob  \n",
       "591  2013073008139  nob  \n",
       "592  2010092803011  nob  \n",
       "\n",
       "[405 rows x 6 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "small_norw = norw_df.loc[norw_df[\"lang\"].isin((\"nob\", \"nno\", \"nor\"))]\n",
    "small_norw"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "But only using language in df_ from the beginning yields 414!\n",
    "This means that 9 of the books that were removed based on filename were actually Norwegian according to the API?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "      <th>filename code</th>\n",
       "      <th>api code</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1856</td>\n",
       "      <td>Barfod, Frederik</td>\n",
       "      <td>Guds Man : en Psalme-Bog</td>\n",
       "      <td>2006111600006</td>\n",
       "      <td>2006111600006</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1852</td>\n",
       "      <td>Fearnley, Henry Emil</td>\n",
       "      <td>Smaadigte til Udenadslæsning : til Brug ved de...</td>\n",
       "      <td>2010031003059</td>\n",
       "      <td>2010031003059</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1852</td>\n",
       "      <td>Hauge, A. Andreas</td>\n",
       "      <td>100 Missions-Psalmer ; med et Tillæg, indehold...</td>\n",
       "      <td>2006111400042</td>\n",
       "      <td>2006111400042</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1863</td>\n",
       "      <td>Hægstad, Marius</td>\n",
       "      <td>Evangelisk-christelig Psalmebog : tilligemed C...</td>\n",
       "      <td>2008121003001</td>\n",
       "      <td>2008121003001</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1898</td>\n",
       "      <td>Skard, Matias</td>\n",
       "      <td>Norsk Digtning efter 1814 : et Udvalg til Brug...</td>\n",
       "      <td>2010081103001</td>\n",
       "      <td>2010081103001</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>588</th>\n",
       "      <td>1925</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Stene i strømmen : digte</td>\n",
       "      <td>2013070308043</td>\n",
       "      <td>2013070308043</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>589</th>\n",
       "      <td>1947</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Samlede dikt</td>\n",
       "      <td>2012082408040</td>\n",
       "      <td>2012082408040</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>590</th>\n",
       "      <td>1950</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Samlede dikt</td>\n",
       "      <td>2011102708060</td>\n",
       "      <td>2011102708060</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>591</th>\n",
       "      <td>1912</td>\n",
       "      <td>Munthe, Margrethe</td>\n",
       "      <td>Aase fiskerpike : eventyrkomedie med sange til...</td>\n",
       "      <td>2013073008139</td>\n",
       "      <td>2013073008139</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>592</th>\n",
       "      <td>1862</td>\n",
       "      <td>Johnsen, J.</td>\n",
       "      <td>En aandelig Sangbog for angerfulde, bodfærdige...</td>\n",
       "      <td>2010092803011</td>\n",
       "      <td>2010092803011</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>414 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     year                author  \\\n",
       "3    1856      Barfod, Frederik   \n",
       "5    1852  Fearnley, Henry Emil   \n",
       "6    1852     Hauge, A. Andreas   \n",
       "10   1863       Hægstad, Marius   \n",
       "13   1898         Skard, Matias   \n",
       "..    ...                   ...   \n",
       "588  1925        Grieg, Nordahl   \n",
       "589  1947        Grieg, Nordahl   \n",
       "590  1950        Grieg, Nordahl   \n",
       "591  1912     Munthe, Margrethe   \n",
       "592  1862           Johnsen, J.   \n",
       "\n",
       "                                                 title  filename code  \\\n",
       "3                             Guds Man : en Psalme-Bog  2006111600006   \n",
       "5    Smaadigte til Udenadslæsning : til Brug ved de...  2010031003059   \n",
       "6    100 Missions-Psalmer ; med et Tillæg, indehold...  2006111400042   \n",
       "10   Evangelisk-christelig Psalmebog : tilligemed C...  2008121003001   \n",
       "13   Norsk Digtning efter 1814 : et Udvalg til Brug...  2010081103001   \n",
       "..                                                 ...            ...   \n",
       "588                           Stene i strømmen : digte  2013070308043   \n",
       "589                                       Samlede dikt  2012082408040   \n",
       "590                                       Samlede dikt  2011102708060   \n",
       "591  Aase fiskerpike : eventyrkomedie med sange til...  2013073008139   \n",
       "592  En aandelig Sangbog for angerfulde, bodfærdige...  2010092803011   \n",
       "\n",
       "          api code lang  \n",
       "3    2006111600006  nob  \n",
       "5    2010031003059  nob  \n",
       "6    2006111400042  nob  \n",
       "10   2008121003001  nob  \n",
       "13   2010081103001  nob  \n",
       "..             ...  ...  \n",
       "588  2013070308043  nob  \n",
       "589  2012082408040  nob  \n",
       "590  2011102708060  nob  \n",
       "591  2013073008139  nob  \n",
       "592  2010092803011  nob  \n",
       "\n",
       "[414 rows x 6 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bigger_norw = df_.loc[df_[\"lang\"].isin((\"nob\", \"nno\", \"nor\"))]\n",
    "bigger_norw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# av de i bigger, hvilke er ikke i smaller?\n",
    "diff_codes = set(bigger_norw[\"filename code\"]) - set(small_norw[\"filename code\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('2008100603009',\n",
       "  'digibok_2008100603009-1851-und-946--Psalmernes_Bog_oversat_efter_Foranstaltning_a.txt'),\n",
       " ('2009041603027',\n",
       "  'digibok_2009041603027-1898-und-891--Udvalgte_digte_for_middelskolen_og_gymnasiet_.txt'),\n",
       " ('2011060804031',\n",
       "  'digibok_2011060804031-1922-und-743-Lindeman_Ludv_M_-Koralbog_indeholdende_de_i_Landstads_Salmebog.txt'),\n",
       " ('2006112000053',\n",
       "  'digibok_2006112000053-1905-mul-955-Lassen_Helene-Selma_Lagerlof_og_hendes_digtning_.txt'),\n",
       " ('2009080603025',\n",
       "  'digibok_2009080603025-1862-und-948-Sinding_Otto-Skovstjerner_en_liden_Digtsamling.txt'),\n",
       " ('2013081206032',\n",
       "  'digibok_2013081206032-1921-und-917-Bull_Jacob_B_-Digte_.txt'),\n",
       " ('2008032702005',\n",
       "  'digibok_2008032702005-1928-und-462-Backe_Stein-Gamle_tufter_dikt.txt'),\n",
       " ('2008040404048',\n",
       "  'digibok_2008040404048-1912-und-970-Eskeland_Lars-Stykke_til_diktat_og_attforteljing_samla_og_t.txt'),\n",
       " ('2012071208112',\n",
       "  'digibok_2012071208112-1911-und-882-Janson_Kristofer-Digte_utgivne_paa_hans_syttiaarige_fodselsdag.txt')]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "code_names = [(get_code(e.name), e.name) for e in poems if get_code(e.name) in diff_codes]    \n",
    "code_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "      <th>filename code</th>\n",
       "      <th>api code</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1856</td>\n",
       "      <td>Barfod, Frederik</td>\n",
       "      <td>Guds Man : en Psalme-Bog</td>\n",
       "      <td>2006111600006</td>\n",
       "      <td>2006111600006</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1852</td>\n",
       "      <td>Fearnley, Henry Emil</td>\n",
       "      <td>Smaadigte til Udenadslæsning : til Brug ved de...</td>\n",
       "      <td>2010031003059</td>\n",
       "      <td>2010031003059</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1852</td>\n",
       "      <td>Hauge, A. Andreas</td>\n",
       "      <td>100 Missions-Psalmer ; med et Tillæg, indehold...</td>\n",
       "      <td>2006111400042</td>\n",
       "      <td>2006111400042</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1863</td>\n",
       "      <td>Hægstad, Marius</td>\n",
       "      <td>Evangelisk-christelig Psalmebog : tilligemed C...</td>\n",
       "      <td>2008121003001</td>\n",
       "      <td>2008121003001</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1898</td>\n",
       "      <td>Skard, Matias</td>\n",
       "      <td>Norsk Digtning efter 1814 : et Udvalg til Brug...</td>\n",
       "      <td>2010081103001</td>\n",
       "      <td>2010081103001</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>588</th>\n",
       "      <td>1925</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Stene i strømmen : digte</td>\n",
       "      <td>2013070308043</td>\n",
       "      <td>2013070308043</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>589</th>\n",
       "      <td>1947</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Samlede dikt</td>\n",
       "      <td>2012082408040</td>\n",
       "      <td>2012082408040</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>590</th>\n",
       "      <td>1950</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Samlede dikt</td>\n",
       "      <td>2011102708060</td>\n",
       "      <td>2011102708060</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>591</th>\n",
       "      <td>1912</td>\n",
       "      <td>Munthe, Margrethe</td>\n",
       "      <td>Aase fiskerpike : eventyrkomedie med sange til...</td>\n",
       "      <td>2013073008139</td>\n",
       "      <td>2013073008139</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>592</th>\n",
       "      <td>1862</td>\n",
       "      <td>Johnsen, J.</td>\n",
       "      <td>En aandelig Sangbog for angerfulde, bodfærdige...</td>\n",
       "      <td>2010092803011</td>\n",
       "      <td>2010092803011</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>414 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     year                author  \\\n",
       "3    1856      Barfod, Frederik   \n",
       "5    1852  Fearnley, Henry Emil   \n",
       "6    1852     Hauge, A. Andreas   \n",
       "10   1863       Hægstad, Marius   \n",
       "13   1898         Skard, Matias   \n",
       "..    ...                   ...   \n",
       "588  1925        Grieg, Nordahl   \n",
       "589  1947        Grieg, Nordahl   \n",
       "590  1950        Grieg, Nordahl   \n",
       "591  1912     Munthe, Margrethe   \n",
       "592  1862           Johnsen, J.   \n",
       "\n",
       "                                                 title  filename code  \\\n",
       "3                             Guds Man : en Psalme-Bog  2006111600006   \n",
       "5    Smaadigte til Udenadslæsning : til Brug ved de...  2010031003059   \n",
       "6    100 Missions-Psalmer ; med et Tillæg, indehold...  2006111400042   \n",
       "10   Evangelisk-christelig Psalmebog : tilligemed C...  2008121003001   \n",
       "13   Norsk Digtning efter 1814 : et Udvalg til Brug...  2010081103001   \n",
       "..                                                 ...            ...   \n",
       "588                           Stene i strømmen : digte  2013070308043   \n",
       "589                                       Samlede dikt  2012082408040   \n",
       "590                                       Samlede dikt  2011102708060   \n",
       "591  Aase fiskerpike : eventyrkomedie med sange til...  2013073008139   \n",
       "592  En aandelig Sangbog for angerfulde, bodfærdige...  2010092803011   \n",
       "\n",
       "          api code lang  \n",
       "3    2006111600006  nob  \n",
       "5    2010031003059  nob  \n",
       "6    2006111400042  nob  \n",
       "10   2008121003001  nob  \n",
       "13   2010081103001  nob  \n",
       "..             ...  ...  \n",
       "588  2013070308043  nob  \n",
       "589  2012082408040  nob  \n",
       "590  2011102708060  nob  \n",
       "591  2013073008139  nob  \n",
       "592  2010092803011  nob  \n",
       "\n",
       "[414 rows x 6 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keep_df = bigger_norw\n",
    "keep_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "    Median: 1890.0\n",
      "    Mean: 1886.9468599033817\n",
      "    Max: 2005\n",
      "    Min: 1790\n",
      "    # of books: 414\n",
      "    \n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "years = [int(year) for year in keep_df[\"year\"]]\n",
    "\n",
    "def print_year_statistics(years):\n",
    "    print(f\"\"\"\n",
    "    Median: {np.median(years)}\n",
    "    Mean: {np.mean(years)}\n",
    "    Max: {np.max(years)}\n",
    "    Min: {np.min(years)}\n",
    "    # of books: {len(years)}\n",
    "    \"\"\")\n",
    "\n",
    "print_year_statistics(years)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Delete unwanted files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "173"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "files_to_delete = df_.loc[~df_[\"lang\"].isin((\"nob\", \"nno\", \"nor\"))]\n",
    "len(files_to_delete)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Count language distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "    Number of bokmål files 380\tor 91.78743961352657%\n",
      "    Number of nynorsk files 30\tor 7.246376811594203%\n",
      "    Number of norsk files 4\tor 0.966183574879227%\n",
      "\n",
      "    \n"
     ]
    }
   ],
   "source": [
    "def print_count_lang_dist(df):\n",
    "\n",
    "    nob = df.loc[df[\"lang\"] == \"nob\"]\n",
    "    nno = df.loc[df[\"lang\"] == \"nno\"]\n",
    "    nor = df.loc[df[\"lang\"] == \"nor\"]\n",
    "\n",
    "    print(f\"\"\"\n",
    "    Number of bokmål files {len(nob)}\\tor {len(nob)/len(df)*100}%\n",
    "    Number of nynorsk files {len(nno)}\\tor {len(nno)/len(df)*100}%\n",
    "    Number of norsk files {len(nor)}\\tor {len(nor)/len(df)*100}%\n",
    "\n",
    "    \"\"\")\n",
    "    \n",
    "print_count_lang_dist(keep_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get metadata for first batch\n",
    "This batch was selected by Språkbanken. Because we got it before we had sorted out irrelevant books the number of received book is higher than we use in our data set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "193\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "      <th>filename code</th>\n",
       "      <th>api code</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1856</td>\n",
       "      <td>Barfod, Frederik</td>\n",
       "      <td>Guds Man : en Psalme-Bog</td>\n",
       "      <td>2006111600006</td>\n",
       "      <td>2006111600006</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1852</td>\n",
       "      <td>Fearnley, Henry Emil</td>\n",
       "      <td>Smaadigte til Udenadslæsning : til Brug ved de...</td>\n",
       "      <td>2010031003059</td>\n",
       "      <td>2010031003059</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1852</td>\n",
       "      <td>Hauge, A. Andreas</td>\n",
       "      <td>100 Missions-Psalmer ; med et Tillæg, indehold...</td>\n",
       "      <td>2006111400042</td>\n",
       "      <td>2006111400042</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1863</td>\n",
       "      <td>Hægstad, Marius</td>\n",
       "      <td>Evangelisk-christelig Psalmebog : tilligemed C...</td>\n",
       "      <td>2008121003001</td>\n",
       "      <td>2008121003001</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1898</td>\n",
       "      <td>Skard, Matias</td>\n",
       "      <td>Norsk Digtning efter 1814 : et Udvalg til Brug...</td>\n",
       "      <td>2010081103001</td>\n",
       "      <td>2010081103001</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>575</th>\n",
       "      <td>1894</td>\n",
       "      <td>Nilsen, Karen</td>\n",
       "      <td>Gransus : nye Digte til beslægtede Sjæle</td>\n",
       "      <td>2013061308130</td>\n",
       "      <td>2013061308130</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>576</th>\n",
       "      <td>1859</td>\n",
       "      <td>Sundt, Johan L.</td>\n",
       "      <td>Christkirken i Nidaros : digtning</td>\n",
       "      <td>2009082103026</td>\n",
       "      <td>2009082103026</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>579</th>\n",
       "      <td>1866</td>\n",
       "      <td>Blom, P.</td>\n",
       "      <td>Aandeligt Strengespil : en Samling Sange, Sukk...</td>\n",
       "      <td>2009090203007</td>\n",
       "      <td>2009090203007</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>581</th>\n",
       "      <td>1854</td>\n",
       "      <td>Bentsen, H.</td>\n",
       "      <td>Lyriske Smaadigte : en Prøvesamling</td>\n",
       "      <td>2009021812003</td>\n",
       "      <td>2009021812003</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>592</th>\n",
       "      <td>1862</td>\n",
       "      <td>Johnsen, J.</td>\n",
       "      <td>En aandelig Sangbog for angerfulde, bodfærdige...</td>\n",
       "      <td>2010092803011</td>\n",
       "      <td>2010092803011</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>193 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     year                author  \\\n",
       "3    1856      Barfod, Frederik   \n",
       "5    1852  Fearnley, Henry Emil   \n",
       "6    1852     Hauge, A. Andreas   \n",
       "10   1863       Hægstad, Marius   \n",
       "13   1898         Skard, Matias   \n",
       "..    ...                   ...   \n",
       "575  1894         Nilsen, Karen   \n",
       "576  1859       Sundt, Johan L.   \n",
       "579  1866              Blom, P.   \n",
       "581  1854           Bentsen, H.   \n",
       "592  1862           Johnsen, J.   \n",
       "\n",
       "                                                 title  filename code  \\\n",
       "3                             Guds Man : en Psalme-Bog  2006111600006   \n",
       "5    Smaadigte til Udenadslæsning : til Brug ved de...  2010031003059   \n",
       "6    100 Missions-Psalmer ; med et Tillæg, indehold...  2006111400042   \n",
       "10   Evangelisk-christelig Psalmebog : tilligemed C...  2008121003001   \n",
       "13   Norsk Digtning efter 1814 : et Udvalg til Brug...  2010081103001   \n",
       "..                                                 ...            ...   \n",
       "575           Gransus : nye Digte til beslægtede Sjæle  2013061308130   \n",
       "576                  Christkirken i Nidaros : digtning  2009082103026   \n",
       "579  Aandeligt Strengespil : en Samling Sange, Sukk...  2009090203007   \n",
       "581                Lyriske Smaadigte : en Prøvesamling  2009021812003   \n",
       "592  En aandelig Sangbog for angerfulde, bodfærdige...  2010092803011   \n",
       "\n",
       "          api code lang  \n",
       "3    2006111600006  nob  \n",
       "5    2010031003059  nob  \n",
       "6    2006111400042  nob  \n",
       "10   2008121003001  nob  \n",
       "13   2010081103001  nob  \n",
       "..             ...  ...  \n",
       "575  2013061308130  nob  \n",
       "576  2009082103026  nob  \n",
       "579  2009090203007  nob  \n",
       "581  2009021812003  nob  \n",
       "592  2010092803011  nob  \n",
       "\n",
       "[193 rows x 6 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p = Path(\"new_ocr/new_ocr_batch1\")\n",
    "\n",
    "codes = [e.name[:-4] for e in p.iterdir()]\n",
    "print(len(codes))\n",
    "\n",
    "fb_df = keep_df.loc[keep_df[\"filename code\"].isin(codes)]\n",
    "fb_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "    Median: 1875.0\n",
      "    Mean: 1875.2072538860104\n",
      "    Max: 1905\n",
      "    Min: 1814\n",
      "    # of books: 193\n",
      "    \n"
     ]
    }
   ],
   "source": [
    "years = [int(year) for year in fb_df[\"year\"]]\n",
    "print_year_statistics(years)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sort on years and get all after 1900\n",
    "Selecting books for batch 2 of re-OCR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "      <th>filename code</th>\n",
       "      <th>api code</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>1920</td>\n",
       "      <td>Klewe, Andreas</td>\n",
       "      <td>Religiøse korsange for sopran, alt, tenor og b...</td>\n",
       "      <td>2012052908151</td>\n",
       "      <td>2012052908151</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>1900</td>\n",
       "      <td>Brun, Yngvar</td>\n",
       "      <td>Europæisk Digtning</td>\n",
       "      <td>2010070206172</td>\n",
       "      <td>2010070206172</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>1914</td>\n",
       "      <td>udgivet af Den Internasjonale bibelstudieforening</td>\n",
       "      <td>Norsk daggry-sangbog</td>\n",
       "      <td>2011051212004</td>\n",
       "      <td>2011051212004</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>1901</td>\n",
       "      <td>Aarrestad, Sven</td>\n",
       "      <td>Sangbog for afholdsforeninger</td>\n",
       "      <td>2012051612001</td>\n",
       "      <td>2012051612001</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>1906</td>\n",
       "      <td>udgivne af Kirke- og undervisningsdepartementet</td>\n",
       "      <td>Fædrelandssange</td>\n",
       "      <td>2006120101031</td>\n",
       "      <td>2006120101031</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>587</th>\n",
       "      <td>1946</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Håbet : dikt</td>\n",
       "      <td>2013012806125</td>\n",
       "      <td>2013012806125</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>588</th>\n",
       "      <td>1925</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Stene i strømmen : digte</td>\n",
       "      <td>2013070308043</td>\n",
       "      <td>2013070308043</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>589</th>\n",
       "      <td>1947</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Samlede dikt</td>\n",
       "      <td>2012082408040</td>\n",
       "      <td>2012082408040</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>590</th>\n",
       "      <td>1950</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Samlede dikt</td>\n",
       "      <td>2011102708060</td>\n",
       "      <td>2011102708060</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>591</th>\n",
       "      <td>1912</td>\n",
       "      <td>Munthe, Margrethe</td>\n",
       "      <td>Aase fiskerpike : eventyrkomedie med sange til...</td>\n",
       "      <td>2013073008139</td>\n",
       "      <td>2013073008139</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>165 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     year                                             author  \\\n",
       "30   1920                                     Klewe, Andreas   \n",
       "36   1900                                       Brun, Yngvar   \n",
       "46   1914  udgivet af Den Internasjonale bibelstudieforening   \n",
       "65   1901                                    Aarrestad, Sven   \n",
       "68   1906    udgivne af Kirke- og undervisningsdepartementet   \n",
       "..    ...                                                ...   \n",
       "587  1946                                     Grieg, Nordahl   \n",
       "588  1925                                     Grieg, Nordahl   \n",
       "589  1947                                     Grieg, Nordahl   \n",
       "590  1950                                     Grieg, Nordahl   \n",
       "591  1912                                  Munthe, Margrethe   \n",
       "\n",
       "                                                 title  filename code  \\\n",
       "30   Religiøse korsange for sopran, alt, tenor og b...  2012052908151   \n",
       "36                                  Europæisk Digtning  2010070206172   \n",
       "46                                Norsk daggry-sangbog  2011051212004   \n",
       "65                       Sangbog for afholdsforeninger  2012051612001   \n",
       "68                                     Fædrelandssange  2006120101031   \n",
       "..                                                 ...            ...   \n",
       "587                                       Håbet : dikt  2013012806125   \n",
       "588                           Stene i strømmen : digte  2013070308043   \n",
       "589                                       Samlede dikt  2012082408040   \n",
       "590                                       Samlede dikt  2011102708060   \n",
       "591  Aase fiskerpike : eventyrkomedie med sange til...  2013073008139   \n",
       "\n",
       "          api code lang  \n",
       "30   2012052908151  nob  \n",
       "36   2010070206172  nob  \n",
       "46   2011051212004  nob  \n",
       "65   2012051612001  nob  \n",
       "68   2006120101031  nob  \n",
       "..             ...  ...  \n",
       "587  2013012806125  nob  \n",
       "588  2013070308043  nob  \n",
       "589  2012082408040  nob  \n",
       "590  2011102708060  nob  \n",
       "591  2013073008139  nob  \n",
       "\n",
       "[165 rows x 6 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "after_1900 = keep_df.loc[keep_df[\"year\"] >= 1900]\n",
    "after_1900"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Count language distribution pre- and post-1900"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AFTER 1900\n",
      "\n",
      "    Number of bokmål files 138\tor 83.63636363636363%\n",
      "    Number of nynorsk files 27\tor 16.363636363636363%\n",
      "    Number of norsk files 0\tor 0.0%\n",
      "\n",
      "    \n",
      "BEFORE 1900\n",
      "\n",
      "    Number of bokmål files 242\tor 97.18875502008032%\n",
      "    Number of nynorsk files 3\tor 1.2048192771084338%\n",
      "    Number of norsk files 4\tor 1.6064257028112447%\n",
      "\n",
      "    \n"
     ]
    }
   ],
   "source": [
    "before_1900 = keep_df.loc[keep_df[\"year\"] < 1900]\n",
    "\n",
    "print(\"AFTER 1900\")\n",
    "print_count_lang_dist(after_1900)\n",
    "\n",
    "print(\"BEFORE 1900\")\n",
    "print_count_lang_dist(before_1900)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Subtract the ones we already have"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Of the 165 books we want, we already have 17\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "      <th>filename code</th>\n",
       "      <th>api code</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>1920</td>\n",
       "      <td>Klewe, Andreas</td>\n",
       "      <td>Religiøse korsange for sopran, alt, tenor og b...</td>\n",
       "      <td>2012052908151</td>\n",
       "      <td>2012052908151</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>1914</td>\n",
       "      <td>udgivet af Den Internasjonale bibelstudieforening</td>\n",
       "      <td>Norsk daggry-sangbog</td>\n",
       "      <td>2011051212004</td>\n",
       "      <td>2011051212004</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>1906</td>\n",
       "      <td>udgivne af Kirke- og undervisningsdepartementet</td>\n",
       "      <td>Fædrelandssange</td>\n",
       "      <td>2006120101031</td>\n",
       "      <td>2006120101031</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>1925</td>\n",
       "      <td>Støylen, Bernt</td>\n",
       "      <td>Nynorsk salmebok : for kyrkja og heim og møte</td>\n",
       "      <td>2010102606030</td>\n",
       "      <td>2010102606030</td>\n",
       "      <td>nno</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>1931</td>\n",
       "      <td>Luther, Martin</td>\n",
       "      <td>Luthers lille katekisme med bibelord og salmevers</td>\n",
       "      <td>2009013000065</td>\n",
       "      <td>2009013000065</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>587</th>\n",
       "      <td>1946</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Håbet : dikt</td>\n",
       "      <td>2013012806125</td>\n",
       "      <td>2013012806125</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>588</th>\n",
       "      <td>1925</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Stene i strømmen : digte</td>\n",
       "      <td>2013070308043</td>\n",
       "      <td>2013070308043</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>589</th>\n",
       "      <td>1947</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Samlede dikt</td>\n",
       "      <td>2012082408040</td>\n",
       "      <td>2012082408040</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>590</th>\n",
       "      <td>1950</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Samlede dikt</td>\n",
       "      <td>2011102708060</td>\n",
       "      <td>2011102708060</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>591</th>\n",
       "      <td>1912</td>\n",
       "      <td>Munthe, Margrethe</td>\n",
       "      <td>Aase fiskerpike : eventyrkomedie med sange til...</td>\n",
       "      <td>2013073008139</td>\n",
       "      <td>2013073008139</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>148 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     year                                             author  \\\n",
       "30   1920                                     Klewe, Andreas   \n",
       "46   1914  udgivet af Den Internasjonale bibelstudieforening   \n",
       "68   1906    udgivne af Kirke- og undervisningsdepartementet   \n",
       "79   1925                                     Støylen, Bernt   \n",
       "84   1931                                     Luther, Martin   \n",
       "..    ...                                                ...   \n",
       "587  1946                                     Grieg, Nordahl   \n",
       "588  1925                                     Grieg, Nordahl   \n",
       "589  1947                                     Grieg, Nordahl   \n",
       "590  1950                                     Grieg, Nordahl   \n",
       "591  1912                                  Munthe, Margrethe   \n",
       "\n",
       "                                                 title  filename code  \\\n",
       "30   Religiøse korsange for sopran, alt, tenor og b...  2012052908151   \n",
       "46                                Norsk daggry-sangbog  2011051212004   \n",
       "68                                     Fædrelandssange  2006120101031   \n",
       "79       Nynorsk salmebok : for kyrkja og heim og møte  2010102606030   \n",
       "84   Luthers lille katekisme med bibelord og salmevers  2009013000065   \n",
       "..                                                 ...            ...   \n",
       "587                                       Håbet : dikt  2013012806125   \n",
       "588                           Stene i strømmen : digte  2013070308043   \n",
       "589                                       Samlede dikt  2012082408040   \n",
       "590                                       Samlede dikt  2011102708060   \n",
       "591  Aase fiskerpike : eventyrkomedie med sange til...  2013073008139   \n",
       "\n",
       "          api code lang  \n",
       "30   2012052908151  nob  \n",
       "46   2011051212004  nob  \n",
       "68   2006120101031  nob  \n",
       "79   2010102606030  nno  \n",
       "84   2009013000065  nob  \n",
       "..             ...  ...  \n",
       "587  2013012806125  nob  \n",
       "588  2013070308043  nob  \n",
       "589  2012082408040  nob  \n",
       "590  2011102708060  nob  \n",
       "591  2013073008139  nob  \n",
       "\n",
       "[148 rows x 6 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "we_need = after_1900.loc[~after_1900[\"filename code\"].isin(fb_df[\"filename code\"])]\n",
    "print(f\"Of the {len(after_1900)} books we want, we already have {len(after_1900)-len(we_need)}\")\n",
    "we_need"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "    Median: 1920.5\n",
      "    Mean: 1926.1756756756756\n",
      "    Max: 2005\n",
      "    Min: 1900\n",
      "    # of books: 148\n",
      "    \n"
     ]
    }
   ],
   "source": [
    "years = we_need[\"year\"]\n",
    "print_year_statistics(years)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check diff we_need and actually received re-OCR\n",
    "We received 138 of the books we wanted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Of the 148 books published after 1900, we got 138, meaning we miss 10.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "      <th>filename code</th>\n",
       "      <th>api code</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>1914</td>\n",
       "      <td>udgivet af Den Internasjonale bibelstudieforening</td>\n",
       "      <td>Norsk daggry-sangbog</td>\n",
       "      <td>2011051212004</td>\n",
       "      <td>2011051212004</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>1925</td>\n",
       "      <td>Støylen, Bernt</td>\n",
       "      <td>Nynorsk salmebok : for kyrkja og heim og møte</td>\n",
       "      <td>2010102606030</td>\n",
       "      <td>2010102606030</td>\n",
       "      <td>nno</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>1931</td>\n",
       "      <td>Luther, Martin</td>\n",
       "      <td>Luthers lille katekisme med bibelord og salmevers</td>\n",
       "      <td>2009013000065</td>\n",
       "      <td>2009013000065</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>1914</td>\n",
       "      <td>Jansen, Henrik</td>\n",
       "      <td>Bergen 1814-1914 : digte</td>\n",
       "      <td>2012022212004</td>\n",
       "      <td>2012022212004</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>1926</td>\n",
       "      <td>Støylen, Bernt</td>\n",
       "      <td>Utgreiding um nynorsk salmebok</td>\n",
       "      <td>2008022704011</td>\n",
       "      <td>2008022704011</td>\n",
       "      <td>nno</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>587</th>\n",
       "      <td>1946</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Håbet : dikt</td>\n",
       "      <td>2013012806125</td>\n",
       "      <td>2013012806125</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>588</th>\n",
       "      <td>1925</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Stene i strømmen : digte</td>\n",
       "      <td>2013070308043</td>\n",
       "      <td>2013070308043</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>589</th>\n",
       "      <td>1947</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Samlede dikt</td>\n",
       "      <td>2012082408040</td>\n",
       "      <td>2012082408040</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>590</th>\n",
       "      <td>1950</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Samlede dikt</td>\n",
       "      <td>2011102708060</td>\n",
       "      <td>2011102708060</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>591</th>\n",
       "      <td>1912</td>\n",
       "      <td>Munthe, Margrethe</td>\n",
       "      <td>Aase fiskerpike : eventyrkomedie med sange til...</td>\n",
       "      <td>2013073008139</td>\n",
       "      <td>2013073008139</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>138 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     year                                             author  \\\n",
       "46   1914  udgivet af Den Internasjonale bibelstudieforening   \n",
       "79   1925                                     Støylen, Bernt   \n",
       "84   1931                                     Luther, Martin   \n",
       "87   1914                                     Jansen, Henrik   \n",
       "91   1926                                     Støylen, Bernt   \n",
       "..    ...                                                ...   \n",
       "587  1946                                     Grieg, Nordahl   \n",
       "588  1925                                     Grieg, Nordahl   \n",
       "589  1947                                     Grieg, Nordahl   \n",
       "590  1950                                     Grieg, Nordahl   \n",
       "591  1912                                  Munthe, Margrethe   \n",
       "\n",
       "                                                 title  filename code  \\\n",
       "46                                Norsk daggry-sangbog  2011051212004   \n",
       "79       Nynorsk salmebok : for kyrkja og heim og møte  2010102606030   \n",
       "84   Luthers lille katekisme med bibelord og salmevers  2009013000065   \n",
       "87                            Bergen 1814-1914 : digte  2012022212004   \n",
       "91                      Utgreiding um nynorsk salmebok  2008022704011   \n",
       "..                                                 ...            ...   \n",
       "587                                       Håbet : dikt  2013012806125   \n",
       "588                           Stene i strømmen : digte  2013070308043   \n",
       "589                                       Samlede dikt  2012082408040   \n",
       "590                                       Samlede dikt  2011102708060   \n",
       "591  Aase fiskerpike : eventyrkomedie med sange til...  2013073008139   \n",
       "\n",
       "          api code lang  \n",
       "46   2011051212004  nob  \n",
       "79   2010102606030  nno  \n",
       "84   2009013000065  nob  \n",
       "87   2012022212004  nob  \n",
       "91   2008022704011  nno  \n",
       "..             ...  ...  \n",
       "587  2013012806125  nob  \n",
       "588  2013070308043  nob  \n",
       "589  2012082408040  nob  \n",
       "590  2011102708060  nob  \n",
       "591  2013073008139  nob  \n",
       "\n",
       "[138 rows x 6 columns]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "p = Path(\"new_ocr/new_ocr_batch2/\")\n",
    "rec_codes = [e.name[:-4] for e in p.iterdir()]\n",
    "\n",
    "rec_df = we_need.loc[we_need[\"filename code\"].isin(rec_codes)]\n",
    "print(f\"Of the {len(we_need)} books published after 1900, we got {len(rec_df)}, meaning we miss {len(we_need)-len(rec_df)}.\")\n",
    "rec_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "rec_df.to_csv(\"batch2_info.tsv\", sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "    Median: 1920.5\n",
      "    Mean: 1925.2173913043478\n",
      "    Max: 1981\n",
      "    Min: 1902\n",
      "    # of books: 138\n",
      "    \n"
     ]
    }
   ],
   "source": [
    "years = rec_df[\"year\"]\n",
    "print_year_statistics(years)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check if any of the books we already annotated are part of the ones we don't need\n",
    "Luckily, this is not the case"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "      <th>filename code</th>\n",
       "      <th>api code</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [year, author, title, filename code, api code, lang]\n",
       "Index: []"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p = Path(\"../../norwegian_rhyme_scheme_corpus/annotation_tool/poems/bokmål\")\n",
    "codes = set([e.name.split(\"_\")[0] for e in p.iterdir()])\n",
    "diff_df.loc[diff_df[\"api code\"].isin(codes)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We annotated 11 books. Of those, 11 are in the ones we need\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "      <th>filename code</th>\n",
       "      <th>api code</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>1931</td>\n",
       "      <td>Luther, Martin</td>\n",
       "      <td>Luthers lille katekisme med bibelord og salmevers</td>\n",
       "      <td>2009013000065</td>\n",
       "      <td>2009013000065</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115</th>\n",
       "      <td>1944</td>\n",
       "      <td>Welhaven, J.S. (Johan Sebastian)</td>\n",
       "      <td>Norges demring : og andre dikt</td>\n",
       "      <td>2012032024065</td>\n",
       "      <td>2012032024065</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>190</th>\n",
       "      <td>1956</td>\n",
       "      <td>Larsen, Alf</td>\n",
       "      <td>Utvalgte dikt</td>\n",
       "      <td>2008010704054</td>\n",
       "      <td>2014102306006</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>386</th>\n",
       "      <td>1914</td>\n",
       "      <td>Bjørnson, Bjørnstjerne</td>\n",
       "      <td>Digte og Sange</td>\n",
       "      <td>2012102608111</td>\n",
       "      <td>2012102608111</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>405</th>\n",
       "      <td>1950</td>\n",
       "      <td>Bjørnson, Bjørnstjerne</td>\n",
       "      <td>Dikte og sange : i utvalg</td>\n",
       "      <td>2006081000055</td>\n",
       "      <td>2006081000055</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>446</th>\n",
       "      <td>1950</td>\n",
       "      <td>Nilsen, Rudolf</td>\n",
       "      <td>Samlede dikt</td>\n",
       "      <td>2011102708082</td>\n",
       "      <td>2011102708082</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>553</th>\n",
       "      <td>1941</td>\n",
       "      <td>Zwilgmeyer, Dagfinn</td>\n",
       "      <td>Nordlandssanger : i herr Petters fotefar</td>\n",
       "      <td>2007011001018</td>\n",
       "      <td>2007011001018</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>558</th>\n",
       "      <td>1934</td>\n",
       "      <td>Holme, Edin</td>\n",
       "      <td>Dikterverker. 4 : Høvdinger ; Tonen fra himlen</td>\n",
       "      <td>2011041408047</td>\n",
       "      <td>2011041408047</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>559</th>\n",
       "      <td>1934</td>\n",
       "      <td>Holme, Edin</td>\n",
       "      <td>Dikterverker. 1 : Ildprofeten ; Menneskesønnen</td>\n",
       "      <td>2011042608056</td>\n",
       "      <td>2011042608056</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>561</th>\n",
       "      <td>1934</td>\n",
       "      <td>Holme, Edin</td>\n",
       "      <td>Dikterverker. 2 : Mannen fra Tarsus</td>\n",
       "      <td>2011041408091</td>\n",
       "      <td>2011041408091</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>587</th>\n",
       "      <td>1946</td>\n",
       "      <td>Grieg, Nordahl</td>\n",
       "      <td>Håbet : dikt</td>\n",
       "      <td>2013012806125</td>\n",
       "      <td>2013012806125</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     year                            author  \\\n",
       "84   1931                    Luther, Martin   \n",
       "115  1944  Welhaven, J.S. (Johan Sebastian)   \n",
       "190  1956                       Larsen, Alf   \n",
       "386  1914            Bjørnson, Bjørnstjerne   \n",
       "405  1950            Bjørnson, Bjørnstjerne   \n",
       "446  1950                    Nilsen, Rudolf   \n",
       "553  1941               Zwilgmeyer, Dagfinn   \n",
       "558  1934                       Holme, Edin   \n",
       "559  1934                       Holme, Edin   \n",
       "561  1934                       Holme, Edin   \n",
       "587  1946                    Grieg, Nordahl   \n",
       "\n",
       "                                                 title  filename code  \\\n",
       "84   Luthers lille katekisme med bibelord og salmevers  2009013000065   \n",
       "115                     Norges demring : og andre dikt  2012032024065   \n",
       "190                                      Utvalgte dikt  2008010704054   \n",
       "386                                     Digte og Sange  2012102608111   \n",
       "405                          Dikte og sange : i utvalg  2006081000055   \n",
       "446                                       Samlede dikt  2011102708082   \n",
       "553           Nordlandssanger : i herr Petters fotefar  2007011001018   \n",
       "558     Dikterverker. 4 : Høvdinger ; Tonen fra himlen  2011041408047   \n",
       "559     Dikterverker. 1 : Ildprofeten ; Menneskesønnen  2011042608056   \n",
       "561                Dikterverker. 2 : Mannen fra Tarsus  2011041408091   \n",
       "587                                       Håbet : dikt  2013012806125   \n",
       "\n",
       "          api code lang  \n",
       "84   2009013000065  nob  \n",
       "115  2012032024065  nob  \n",
       "190  2014102306006  nob  \n",
       "386  2012102608111  nob  \n",
       "405  2006081000055  nob  \n",
       "446  2011102708082  nob  \n",
       "553  2007011001018  nob  \n",
       "558  2011041408047  nob  \n",
       "559  2011042608056  nob  \n",
       "561  2011041408091  nob  \n",
       "587  2013012806125  nob  "
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hmm = we_need.loc[we_need[\"api code\"].isin(codes)]\n",
    "print(f\"We annotated {len(codes)} books. Of those, {len(hmm)} are in the ones we need\")\n",
    "hmm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check that examples from thesis are actually in our selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "      <th>filename code</th>\n",
       "      <th>api code</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>1893</td>\n",
       "      <td>Aagaard, Gustav</td>\n",
       "      <td>Sjøluft : billeder i digt</td>\n",
       "      <td>2009022603004</td>\n",
       "      <td>2009022603004</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>1857</td>\n",
       "      <td>Heyerdahl, Halvor</td>\n",
       "      <td>Historiske Sange fra Norges Fortid</td>\n",
       "      <td>2010030903071</td>\n",
       "      <td>2010030903071</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     year             author                               title  \\\n",
       "89   1893    Aagaard, Gustav           Sjøluft : billeder i digt   \n",
       "109  1857  Heyerdahl, Halvor  Historiske Sange fra Norges Fortid   \n",
       "\n",
       "     filename code       api code lang  \n",
       "89   2009022603004  2009022603004  nob  \n",
       "109  2010030903071  2010030903071  nob  "
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hustru = \"2009022603004\" \n",
    "voler = \"2010030903071\"\n",
    "keep_df.loc[keep_df[\"api code\"].isin([hustru, voler])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "      <th>filename code</th>\n",
       "      <th>api code</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>1893</td>\n",
       "      <td>Aagaard, Gustav</td>\n",
       "      <td>Sjøluft : billeder i digt</td>\n",
       "      <td>2009022603004</td>\n",
       "      <td>2009022603004</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>1857</td>\n",
       "      <td>Heyerdahl, Halvor</td>\n",
       "      <td>Historiske Sange fra Norges Fortid</td>\n",
       "      <td>2010030903071</td>\n",
       "      <td>2010030903071</td>\n",
       "      <td>nob</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     year             author                               title  \\\n",
       "89   1893    Aagaard, Gustav           Sjøluft : billeder i digt   \n",
       "109  1857  Heyerdahl, Halvor  Historiske Sange fra Norges Fortid   \n",
       "\n",
       "     filename code       api code lang  \n",
       "89   2009022603004  2009022603004  nob  \n",
       "109  2010030903071  2010030903071  nob  "
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fb_df.loc[fb_df[\"api code\"].isin([hustru, voler])]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gather received re-OCR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "re_ocr = pd.concat((rec_df, fb_df))\n",
    "pre_1900 = re_ocr.loc[re_ocr[\"year\"] < 1900]\n",
    "post_1900 = re_ocr.loc[re_ocr[\"year\"] >= 1900]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "re_ocr.to_csv(\"re-ocr_info.tsv\", sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FULL re-OCR\n",
      "\n",
      "    Median: 1897.0\n",
      "    Mean: 1896.0574018126888\n",
      "    Max: 1981\n",
      "    Min: 1814\n",
      "    # of books: 331\n",
      "    \n",
      "\n",
      "    Number of bokmål files 300\tor 90.6344410876133%\n",
      "    Number of nynorsk files 30\tor 9.06344410876133%\n",
      "    Number of norsk files 1\tor 0.3021148036253776%\n",
      "\n",
      "    \n"
     ]
    }
   ],
   "source": [
    "print(\"FULL re-OCR\")\n",
    "years = re_ocr[\"year\"]\n",
    "print_year_statistics(years)\n",
    "print_count_lang_dist(re_ocr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PRE-1900 re-OCR\n",
      "\n",
      "    Median: 1871.5\n",
      "    Mean: 1872.6761363636363\n",
      "    Max: 1899\n",
      "    Min: 1814\n",
      "    # of books: 176\n",
      "    \n",
      "\n",
      "    Number of bokmål files 172\tor 97.72727272727273%\n",
      "    Number of nynorsk files 3\tor 1.7045454545454544%\n",
      "    Number of norsk files 1\tor 0.5681818181818182%\n",
      "\n",
      "    \n"
     ]
    }
   ],
   "source": [
    "print(\"PRE-1900 re-OCR\")\n",
    "years = pre_1900[\"year\"]\n",
    "print_year_statistics(years)\n",
    "print_count_lang_dist(pre_1900)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "POST-1900 re-OCR\n",
      "\n",
      "    Median: 1920.0\n",
      "    Mean: 1922.6064516129031\n",
      "    Max: 1981\n",
      "    Min: 1900\n",
      "    # of books: 155\n",
      "    \n",
      "\n",
      "    Number of bokmål files 128\tor 82.58064516129032%\n",
      "    Number of nynorsk files 27\tor 17.419354838709676%\n",
      "    Number of norsk files 0\tor 0.0%\n",
      "\n",
      "    \n"
     ]
    }
   ],
   "source": [
    "print(\"POST-1900 re-OCR\")\n",
    "years = post_1900[\"year\"]\n",
    "print_year_statistics(years)\n",
    "print_count_lang_dist(post_1900)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "331\n"
     ]
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "p1 = Path(\"new_ocr/new_ocr_batch1/\")\n",
    "p2 = Path(\"new_ocr/new_ocr_batch2/\")\n",
    "\n",
    "print(len(list(p1.iterdir())) + len(list(p2.iterdir())))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}