0.1-setup.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ライブドアニュースコーパスからデータ作成\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "tgz_fname = \"ldcc-20140209.tar.gz\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 分類用データへの加工\n",
    "\n",
    "* tar.gzファイルから2つのジャンルを対象にする\n",
    "  * text/GENRE/GENRE-#######.txt\n",
    "    * 3行目がタイトルなのでこれを分類対象とする\n",
    "      * 文章の前後についている【】を削除\n",
    "* tsvファイルとして出力\n",
    "  * フィールド構造\n",
    "    1. (未使用)\n",
    "    2. クラス (0/1)\n",
    "    3. (未使用)\n",
    "    4. テキスト"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "! mkdir swp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tarfile\n",
    "import csv\n",
    "import re\n",
    "\n",
    "target_genre = [\"sports-watch\", \"peachy\"] # 大きく異なるジャンル2つを選定\n",
    "\n",
    "zero_fnames = []\n",
    "one_fnames = []\n",
    "tsv_fname = \"swp/all.tsv\"\n",
    "\n",
    "brackets_tail = re.compile('【[^】]*】$')\n",
    "brackets_head = re.compile('^【[^】]*】')\n",
    "\n",
    "def remove_brackets(inp):\n",
    "    output = re.sub(brackets_head, '',\n",
    "                   re.sub(brackets_tail, '', inp))\n",
    "    return output\n",
    "\n",
    "def read_title(f):\n",
    "    # 2行スキップ\n",
    "    next(f)\n",
    "    next(f)\n",
    "    title = next(f) # 3行目を返す\n",
    "    title = remove_brackets(title.decode('utf-8'))\n",
    "    return title[:-1]\n",
    "\n",
    "with tarfile.open(tgz_fname) as tf:\n",
    "    # 対象ファイルの選定\n",
    "    for ti in tf:\n",
    "        # ライセンスファイルはスキップ\n",
    "        if \"LICENSE.txt\" in ti.name:\n",
    "            continue\n",
    "        if target_genre[0] in ti.name and ti.name.endswith(\".txt\"):\n",
    "            zero_fnames.append(ti.name)\n",
    "            continue\n",
    "        if target_genre[1] in ti.name and ti.name.endswith(\".txt\"):\n",
    "            one_fnames.append(ti.name)\n",
    "    with open(tsv_fname, \"w\") as wf:\n",
    "        writer = csv.writer(wf, delimiter='\\t')\n",
    "        # ラベル 0\n",
    "        for name in zero_fnames:\n",
    "            f = tf.extractfile(name)\n",
    "            title = read_title(f)\n",
    "            row = [target_genre[0], 0, '', title]\n",
    "            writer.writerow(row)\n",
    "        # ラベル 1\n",
    "        for name in one_fnames:\n",
    "            f = tf.extractfile(name)\n",
    "            title = read_title(f)\n",
    "            row = [target_genre[1], 1, '', title]\n",
    "            writer.writerow(row)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# データのシャッフルと分割\n",
    "\n",
    "* train/dev/test : 8:1:1 で分割"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "random.seed(100)\n",
    "with open(\"swp/all.tsv\", 'r') as f, open(\"swp/rand-all.tsv\", \"w\") as wf:\n",
    "    lines = f.readlines()\n",
    "    random.shuffle(lines)\n",
    "    for line in lines:\n",
    "        wf.write(line)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(101)\n",
    "\n",
    "train_fname, dev_fname, test_fname = [\"swp/train.tsv\", \"swp/dev.tsv\", \"swp/test.tsv\"]\n",
    "\n",
    "with open(\"swp/rand-all.tsv\") as f, open(train_fname, \"w\") as tf, open(dev_fname, \"w\") as df, open(test_fname, \"w\") as ef:\n",
    "    ef.write(\"class\\tsentence\\n\")\n",
    "    for line in f:\n",
    "        v = random.randint(0, 9)\n",
    "        if v == 8:\n",
    "            df.write(line)\n",
    "        elif v == 9:\n",
    "            row = line.split('\\t')\n",
    "            ef.write(\"\\t\".join([row[1], row[3]]))\n",
    "        else:\n",
    "            tf.write(line)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "BERTモデルのセットアップは [0-setup](0-setup.ipynb)を参照"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}