4-aozora-no-checkpoint.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "WARNING:tensorflow:Estimator's model_fn (<function model_fn_builder.<locals>.model_fn at 0x7fea3d567598>) includes params argument, but params are not passed to Estimator.\n",
      "INFO:tensorflow:Using config: {'_model_dir': 'aozora/cola_3_output_nockpt', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 1000, '_save_checkpoints_secs': None, '_session_config': None, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7fea2d3e49e8>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=1000, num_shards=8, num_cores_per_replica=None, per_host_input_for_training=3, tpu_job_name=None, initial_infeed_sleep_secs=None), '_cluster': None}\n",
      "INFO:tensorflow:_TPUContext: eval_on_tpu True\n",
      "WARNING:tensorflow:eval_on_tpu ignored because use_tpu is False.\n",
      "INFO:tensorflow:Writing example 0 of 9746\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: train-0\n",
      "INFO:tensorflow:tokens: [CLS] それ ##から 春 になって ##チ ##ュ ##ン ##セ ##は 学 校 も 六 年 て ##さ ##か ##って ##しまい ##ま ##した 。 チ ##ュ ##ン ##セ ##は ##も ##う 働 いている ##のて ##す 。 春 に 、 く ##る ##み ##の 木 か ##み ##んな 青 い 房 のような ##ものを 下 けている ##て ##し ##ょう 。 その 下 に ##し ##ゃ ##か ##ん ##て 、 チ ##ュ ##ン ##セ ##は ##キ ##ャ ##ヘ ##シ ##の 床 を ##つ ##く ##って ##い ##ま ##した 。 そ ##した ##ら 土 の 中 から 一 ひ ##き ##の ##う ##す ##い 緑 い ##ろ ##の 小 さな 蛙 か ##よ ##ろ ##よ ##ろ ##と 這 って 出 て 来 ま ##した 。 [SEP]\n",
      "INFO:tensorflow:input_ids: 101 48885 17587 4348 95110 31192 81095 12426 55175 11572 3171 4579 1547 2078 3498 1531 44180 10845 26433 67965 26397 17870 1483 1586 81095 12426 55175 11572 13639 23543 2012 33981 27145 16018 1483 4348 1534 1482 1519 16206 17717 10711 4462 1517 17717 58123 9262 1510 3927 32898 52999 1658 35167 11883 14440 80972 1483 11547 1658 1534 14440 97384 10845 18378 11883 1482 1586 81095 12426 55175 11572 26468 68110 97394 25499 10711 3514 1562 23393 17461 26433 14811 26397 17870 1483 1526 17870 17044 2767 1537 1683 11037 1651 1539 16181 10711 23543 16018 14811 6633 1510 37243 10711 3273 59473 7545 1517 46731 37243 46731 37243 11651 8473 15188 2157 1531 4495 1543 17870 1483 102 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: 1 (id = 1)\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: train-1\n",
      "INFO:tensorflow:tokens: [CLS] 「 う ##ん 、 と ##う ##も ##あ ##ふ ##ない ##と 僕 も 思 った 。 こ ##っ ##ち ##は 止 そ ##う 。 と ##って ##し ##ま ##お ##う 。 その 辺 へ ##か ##く ##して ##お ##いて ##あ ##と ##て 我 われ ##か ##と ##った ##という ##ことに ##して ##お 嬢 さん ##に ##ても 上 け ##れは ##い ##い ##し ##ゃ ##ないか 。 その ##ほ ##う ##か 安 全 た ##よ 。 」 という ##のか ##は ##っ ##き ##り 聞 え ##ま ##した 。 私 たちは ##また 顔 を 見 合 せ ##ま ##した 。 [SEP]\n",
      "INFO:tensorflow:input_ids: 101 1492 1512 18378 1482 1532 23543 13639 97378 75049 15295 11651 2015 1547 3709 12563 1483 1521 97383 29644 11572 4846 1526 23543 1483 1532 26433 14440 26397 24851 23543 1483 11547 8413 1541 10845 17461 22150 24851 29512 97378 11651 11883 3901 28967 10845 11651 31150 17606 30659 22150 24851 3143 43437 11488 40167 1657 1520 29386 14811 14811 14440 97384 90973 1483 11547 62040 23543 10845 3189 2073 1527 46731 1483 1493 14166 25257 11572 97383 16181 15500 6914 1514 26397 17870 1483 6260 72521 52479 9357 1562 7753 2429 1525 26397 17870 1483 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: 1 (id = 1)\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: train-2\n",
      "INFO:tensorflow:tokens: [CLS] 赤 狐 は 扉 を ##しめ ##て 云 ひ ##ま ##した 。 僕 は ##あ ##の ##い ##つ ##か 公 爵 の 子 供 か 着 て 居 た 赤 い 上 着 な ##ら 見 た ##い ##な ##あ ##と 仔 牛 は 思 ひ ##ま ##した ##け ##れ ##とも ##も ##う 狐 か ##く ##ん ##く ##ん 向 ふ ##へ 行 く ##も ##ん ##て ##す ##から 仕 方 なく ##つ ##いて 行 き ##ま ##した 。 [SEP]\n",
      "INFO:tensorflow:input_ids: 101 8212 5608 1538 3933 1562 69106 11883 1735 1539 26397 17870 1483 2015 1538 97378 10711 14811 23393 10845 2077 5549 1537 3155 1885 1517 6043 1531 3301 1527 8212 1510 1657 6043 1533 17044 7753 1527 14811 22353 97378 11651 1780 5567 1538 3709 1539 26397 17870 23024 28028 54772 13639 23543 5608 1517 17461 18378 17461 18378 2438 1540 43458 7658 1519 13639 18378 11883 16018 17587 1781 4301 57516 23393 29512 7658 1518 26397 17870 1483 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: 1 (id = 1)\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: train-3\n",
      "INFO:tensorflow:tokens: [CLS] 俊 助 は 制 服 の 隠 し ##を ##さ ##く ##って 、 名 刺 を 辰 子 の 手 へ 渡 した 。 渡 す 時 向 う ##の 手 を 見 ると 、 青 玉 を 入 れた 金 の 指 環 か 、 細 っ ##そ ##り ##と ##その 小 指 を 繞 っていた 。 俊 助 は ##そ ##れ ##も ##また 美 しい ##と 思 った 。 [SEP]\n",
      "INFO:tensorflow:input_ids: 101 1908 2242 1538 2189 4451 1537 9182 1523 11465 44180 17461 26433 1482 2434 2193 1562 8408 3155 1537 3934 1541 5184 10568 1483 5184 1524 4363 2438 1512 10711 3934 1562 7753 43110 1482 9262 5675 1562 2071 21618 8688 1537 4037 5800 1517 1482 6577 1529 69175 15500 11651 19504 3273 4037 1562 6682 24654 1483 1908 2242 1538 69175 28028 13639 52479 6839 18658 11651 3709 12563 1483 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: 0 (id = 0)\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: train-4\n",
      "INFO:tensorflow:tokens: [CLS] 最 後 に 小 泉 孤 松 の 書 いた 「 農 家 義 人 伝 」 の 中 の 一 篇 によれは 、 平 四 郎 は 伝 吉 の 牽 いていた 馬 に 泥 田 へ 蹴 落 された ##と 云 う ##ことて ##ある 。 （ 註 三 ） [SEP]\n",
      "INFO:tensorflow:input_ids: 101 4446 3651 1534 3273 5016 3170 4507 1537 4439 17210 1492 8410 3219 6850 1763 1818 1493 1537 1683 1537 1651 6451 53840 1482 3497 2733 8575 1538 1818 2430 1537 5582 67500 9492 1534 5029 5841 1541 8285 7339 11105 11651 1735 1512 24230 17977 1483 10028 7812 1656 10029 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: 0 (id = 0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:***** Running training *****\n",
      "INFO:tensorflow:  Num examples = 9746\n",
      "INFO:tensorflow:  Batch size = 32\n",
      "INFO:tensorflow:  Num steps = 913\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Running train on CPU\n",
      "INFO:tensorflow:*** Features ***\n",
      "INFO:tensorflow:  name = input_ids, shape = (32, 128)\n",
      "INFO:tensorflow:  name = input_mask, shape = (32, 128)\n",
      "INFO:tensorflow:  name = label_ids, shape = (32,)\n",
      "INFO:tensorflow:  name = segment_ids, shape = (32, 128)\n",
      "INFO:tensorflow:**** Trainable Variables ****\n",
      "INFO:tensorflow:  name = bert/embeddings/word_embeddings:0, shape = (105879, 768)\n",
      "INFO:tensorflow:  name = bert/embeddings/token_type_embeddings:0, shape = (2, 768)\n",
      "INFO:tensorflow:  name = bert/embeddings/position_embeddings:0, shape = (512, 768)\n",
      "INFO:tensorflow:  name = bert/embeddings/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/embeddings/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/pooler/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/pooler/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = output_weights:0, shape = (2, 768)\n",
      "INFO:tensorflow:  name = output_bias:0, shape = (2,)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "2018-11-21 06:33:17.441816: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n",
      "2018-11-21 06:33:17.519024: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:897] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2018-11-21 06:33:17.519412: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1405] Found device 0 with properties: \n",
      "name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235\n",
      "pciBusID: 0000:00:1e.0\n",
      "totalMemory: 11.17GiB freeMemory: 11.10GiB\n",
      "2018-11-21 06:33:17.519440: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1484] Adding visible gpu devices: 0\n",
      "2018-11-21 06:33:17.800720: I tensorflow/core/common_runtime/gpu/gpu_device.cc:965] Device interconnect StreamExecutor with strength 1 edge matrix:\n",
      "2018-11-21 06:33:17.800772: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971]      0 \n",
      "2018-11-21 06:33:17.800792: I tensorflow/core/common_runtime/gpu/gpu_device.cc:984] 0:   N \n",
      "2018-11-21 06:33:17.801082: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1097] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10758 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:1e.0, compute capability: 3.7)\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into aozora/cola_3_output_nockpt/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.634307\n",
      "INFO:tensorflow:examples/sec: 20.2978\n",
      "INFO:tensorflow:global_step/sec: 0.646301\n",
      "INFO:tensorflow:examples/sec: 20.6816\n",
      "INFO:tensorflow:global_step/sec: 0.645283\n",
      "INFO:tensorflow:examples/sec: 20.6491\n",
      "INFO:tensorflow:global_step/sec: 0.644978\n",
      "INFO:tensorflow:examples/sec: 20.6393\n",
      "INFO:tensorflow:global_step/sec: 0.645356\n",
      "INFO:tensorflow:examples/sec: 20.6514\n",
      "INFO:tensorflow:global_step/sec: 0.645283\n",
      "INFO:tensorflow:examples/sec: 20.649\n",
      "INFO:tensorflow:global_step/sec: 0.644995\n",
      "INFO:tensorflow:examples/sec: 20.6399\n",
      "INFO:tensorflow:global_step/sec: 0.645106\n",
      "INFO:tensorflow:examples/sec: 20.6434\n",
      "INFO:tensorflow:global_step/sec: 0.64504\n",
      "INFO:tensorflow:examples/sec: 20.6413\n",
      "INFO:tensorflow:Saving checkpoints for 913 into aozora/cola_3_output_nockpt/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.0020948937.\n",
      "INFO:tensorflow:Writing example 0 of 1195\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: dev-0\n",
      "INFO:tensorflow:tokens: [CLS] （ 何 箇 月 か ##たった 後 、 僕 は 何 か ##の 話 の 次 手 に 『 悪 魔 』 の 作 家 に 彼 の 言 葉 を 話 した 。 すると ##この 作 家 は 笑 い ##なから 、 無 造 作 に 僕 に ##こ ##う 言 う ##の ##たった 。 [UNK] [UNK] 「 世 界 一 な ##らは 何 ても 好 い 。 」 ！ ） [SEP]\n",
      "INFO:tensorflow:input_ids: 101 10028 1847 6427 4448 1517 57603 3651 1482 2015 1538 1847 1517 10711 7836 1537 4827 3934 1534 1494 3767 9646 1495 1537 1853 3219 1534 3640 1537 7785 7341 1562 7836 10568 1483 20969 20864 1853 3219 1538 6381 1510 32980 1482 5466 8479 1853 1534 2015 1534 23635 23543 7785 1512 10711 57603 1483 100 100 1492 1666 5855 1651 1533 46775 1847 16432 3019 1510 1483 1493 10023 10029 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: 0 (id = 0)\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: dev-1\n",
      "INFO:tensorflow:tokens: [CLS] 林 中 の 女 の ##ふ ##く ##ろう ##か ##ま ##る ##て 口 口 に 答 え ##ま ##した 。 その 音 は 二 町 は ##かり 西 の 方 の 大 きな 藁 屋 根 の 中 に 捕 われている 穂 吉 の 処 まて 、 ほ ##ん ##のか ##す ##かに ##て ##した ##け ##れ ##とも 聞 えた ##のて ##す 。 [SEP]\n",
      "INFO:tensorflow:input_ids: 101 4516 1683 1537 3014 1537 75049 17461 60422 10845 26397 16206 11883 2404 2404 1534 6410 1514 26397 17870 1483 11547 9313 1538 1732 5848 1538 50663 7748 1537 4301 1537 2974 24725 7459 3305 4588 1537 1683 1534 4071 29304 6307 2430 1537 2145 14579 1482 1542 18378 25257 16018 86250 11883 17870 23024 28028 54772 6914 19542 27145 16018 1483 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: 1 (id = 1)\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: dev-2\n",
      "INFO:tensorflow:tokens: [CLS] 廣 い 門 の 下 には 、 この 男 の 外 に 誰 も ##ゐ ##ない 。 唯 、 所 々の 剥 けた 、 大 きな 圓 柱 に 、 蟋 蟀 か 一 匹 と ##ま ##つ ##て ##ゐ ##る 。 羅 生 門 か 、 朱 雀 大 路 にある 以 上 は 、 この 男 の 外 にも 、 雨 や ##み ##を ##する 市 女 笠 や 揉 烏 帽 子 か 、 もう 二 三 は ##あり ##さ ##う ##な ##ものてある 。 それ ##か 、 この 男 の 外 には 誰 も ##ゐ ##ない 。 [SEP]\n",
      "INFO:tensorflow:input_ids: 101 3559 1510 9028 1537 1658 11350 1482 12058 5846 1537 2965 1534 7863 1547 97388 15295 1483 2578 1482 3928 39522 2213 17052 1482 2974 24725 2763 4555 1534 1482 7626 7622 1517 1651 2308 1532 26397 23393 11883 97388 16206 1483 6833 5830 9028 1517 1482 4469 9198 2974 8252 17372 1792 1657 1538 1482 12058 5846 1537 2965 15599 1482 9221 1549 17717 11465 28402 3451 3014 6387 1549 4125 5435 3480 3155 1517 1482 89828 1732 1656 1538 29189 44180 23543 22353 35892 1483 48885 10845 1482 12058 5846 1537 2965 11350 7863 1547 97388 15295 1483 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: 0 (id = 0)\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: dev-3\n",
      "INFO:tensorflow:tokens: [CLS] 「 あ ##あ 、 そ ##う ##て ##す ##か 、 そ ##い ##し ##ゃ 私 の ##ほ ##う ##か ##や ##っ ##は ##り 詳 しく 知 って ##ます 。 この 間 まて 馬 喰 を ##や ##って ##ま ##した ##か ##ね 。 今 こ ##ろ ##は 何 をしている ##か 全 く 困 った ##も ##ん ##て ##す ##よ 。 」 [SEP]\n",
      "INFO:tensorflow:input_ids: 101 1492 1508 97378 1482 1526 23543 11883 16018 10845 1482 1526 14811 14440 97384 6260 1537 62040 23543 10845 20936 97383 11572 15500 7838 23818 6091 15188 46630 1483 12058 9038 14579 9492 2629 1562 20936 26433 26397 17870 10845 61828 1483 1773 1521 37243 11572 1847 57483 10845 2073 1519 2745 12563 13639 18378 11883 16018 46731 1483 1493 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: 1 (id = 1)\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: dev-4\n",
      "INFO:tensorflow:tokens: [CLS] 私 は 覚 束 ない 声 を 出 して 、 何 と 云 う 事 も ##なく ##こ ##う 問 い 返 し ##ま ##した 。 実 際 その 時 の 私 には 、 また 摩 利 信 乃 法 師 を 殺 そ ##う ##とも 、 殺 す ##ま ##い ##とも 、 は ##っ ##き ##り ##した 決 断 かつ ##か ##すに ##いた ##のて ##こ ##さ ##い ##ます 。 か 、 そ ##う 云 う 内 にも 私 の 甥 か 、 今 度 は ##ふ ##り 向 く ##ら ##しい 容 子 も ##なく 、 し ##っと ##その 小 屋 を 見 守 り ##なから 、 [SEP]\n",
      "INFO:tensorflow:input_ids: 101 6260 1538 7758 4490 35141 2949 1562 2157 11656 1482 1847 1532 1735 1512 1731 1547 26557 23635 23543 2589 1510 8428 1523 26397 17870 1483 3204 9180 11547 4363 1537 6260 11350 1482 12141 4173 2183 1920 1696 5020 3470 1562 4885 1526 23543 54772 1482 4885 1524 26397 14811 54772 1482 1538 97383 16181 15500 17870 4975 4297 59750 10845 93133 23876 27145 23635 44180 14811 46630 1483 1517 1482 1526 23543 1735 1512 2093 15599 6260 1537 5834 1517 1482 1773 3530 1538 75049 15500 2438 1519 17044 37237 3221 3155 1547 26557 1482 1523 69416 19504 3273 3305 1562 7753 3188 1555 32980 1482 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: 0 (id = 0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:***** Running evaluation *****\n",
      "INFO:tensorflow:  Num examples = 1195\n",
      "INFO:tensorflow:  Batch size = 8\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Running eval on CPU\n",
      "INFO:tensorflow:*** Features ***\n",
      "INFO:tensorflow:  name = input_ids, shape = (?, 128)\n",
      "INFO:tensorflow:  name = input_mask, shape = (?, 128)\n",
      "INFO:tensorflow:  name = label_ids, shape = (?,)\n",
      "INFO:tensorflow:  name = segment_ids, shape = (?, 128)\n",
      "INFO:tensorflow:**** Trainable Variables ****\n",
      "INFO:tensorflow:  name = bert/embeddings/word_embeddings:0, shape = (105879, 768)\n",
      "INFO:tensorflow:  name = bert/embeddings/token_type_embeddings:0, shape = (2, 768)\n",
      "INFO:tensorflow:  name = bert/embeddings/position_embeddings:0, shape = (512, 768)\n",
      "INFO:tensorflow:  name = bert/embeddings/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/embeddings/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/pooler/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/pooler/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = output_weights:0, shape = (2, 768)\n",
      "INFO:tensorflow:  name = output_bias:0, shape = (2,)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2018-11-21-06:58:04\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "2018-11-21 06:58:04.980610: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1484] Adding visible gpu devices: 0\n",
      "2018-11-21 06:58:04.980669: I tensorflow/core/common_runtime/gpu/gpu_device.cc:965] Device interconnect StreamExecutor with strength 1 edge matrix:\n",
      "2018-11-21 06:58:04.980686: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971]      0 \n",
      "2018-11-21 06:58:04.980702: I tensorflow/core/common_runtime/gpu/gpu_device.cc:984] 0:   N \n",
      "2018-11-21 06:58:04.980840: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1097] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10758 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:1e.0, compute capability: 3.7)\n",
      "INFO:tensorflow:Restoring parameters from aozora/cola_3_output_nockpt/model.ckpt-913\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2018-11-21-06:58:35\n",
      "INFO:tensorflow:Saving dict for global step 913: eval_accuracy = 0.98159, eval_loss = 0.069068, global_step = 913, loss = 0.06878654\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 913: aozora/cola_3_output_nockpt/model.ckpt-913\n",
      "INFO:tensorflow:***** Eval results *****\n",
      "INFO:tensorflow:  eval_accuracy = 0.98159\n",
      "INFO:tensorflow:  eval_loss = 0.069068\n",
      "INFO:tensorflow:  global_step = 913\n",
      "INFO:tensorflow:  loss = 0.06878654\n"
     ]
    }
   ],
   "source": [
    "# epoch 3 without checkpoint\n",
    "! ./aozora_cola.sh -e 3 -p /opt/p3/bin/python -n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "WARNING:tensorflow:Estimator's model_fn (<function model_fn_builder.<locals>.model_fn at 0x7f593d627598>) includes params argument, but params are not passed to Estimator.\n",
      "INFO:tensorflow:Using config: {'_model_dir': 'aozora/cola_10_output_nockpt', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 1000, '_save_checkpoints_secs': None, '_session_config': None, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f592d4a4978>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=1000, num_shards=8, num_cores_per_replica=None, per_host_input_for_training=3, tpu_job_name=None, initial_infeed_sleep_secs=None), '_cluster': None}\n",
      "INFO:tensorflow:_TPUContext: eval_on_tpu True\n",
      "WARNING:tensorflow:eval_on_tpu ignored because use_tpu is False.\n",
      "INFO:tensorflow:Writing example 0 of 9746\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: train-0\n",
      "INFO:tensorflow:tokens: [CLS] それ ##から 春 になって ##チ ##ュ ##ン ##セ ##は 学 校 も 六 年 て ##さ ##か ##って ##しまい ##ま ##した 。 チ ##ュ ##ン ##セ ##は ##も ##う 働 いている ##のて ##す 。 春 に 、 く ##る ##み ##の 木 か ##み ##んな 青 い 房 のような ##ものを 下 けている ##て ##し ##ょう 。 その 下 に ##し ##ゃ ##か ##ん ##て 、 チ ##ュ ##ン ##セ ##は ##キ ##ャ ##ヘ ##シ ##の 床 を ##つ ##く ##って ##い ##ま ##した 。 そ ##した ##ら 土 の 中 から 一 ひ ##き ##の ##う ##す ##い 緑 い ##ろ ##の 小 さな 蛙 か ##よ ##ろ ##よ ##ろ ##と 這 って 出 て 来 ま ##した 。 [SEP]\n",
      "INFO:tensorflow:input_ids: 101 48885 17587 4348 95110 31192 81095 12426 55175 11572 3171 4579 1547 2078 3498 1531 44180 10845 26433 67965 26397 17870 1483 1586 81095 12426 55175 11572 13639 23543 2012 33981 27145 16018 1483 4348 1534 1482 1519 16206 17717 10711 4462 1517 17717 58123 9262 1510 3927 32898 52999 1658 35167 11883 14440 80972 1483 11547 1658 1534 14440 97384 10845 18378 11883 1482 1586 81095 12426 55175 11572 26468 68110 97394 25499 10711 3514 1562 23393 17461 26433 14811 26397 17870 1483 1526 17870 17044 2767 1537 1683 11037 1651 1539 16181 10711 23543 16018 14811 6633 1510 37243 10711 3273 59473 7545 1517 46731 37243 46731 37243 11651 8473 15188 2157 1531 4495 1543 17870 1483 102 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: 1 (id = 1)\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: train-1\n",
      "INFO:tensorflow:tokens: [CLS] 「 う ##ん 、 と ##う ##も ##あ ##ふ ##ない ##と 僕 も 思 った 。 こ ##っ ##ち ##は 止 そ ##う 。 と ##って ##し ##ま ##お ##う 。 その 辺 へ ##か ##く ##して ##お ##いて ##あ ##と ##て 我 われ ##か ##と ##った ##という ##ことに ##して ##お 嬢 さん ##に ##ても 上 け ##れは ##い ##い ##し ##ゃ ##ないか 。 その ##ほ ##う ##か 安 全 た ##よ 。 」 という ##のか ##は ##っ ##き ##り 聞 え ##ま ##した 。 私 たちは ##また 顔 を 見 合 せ ##ま ##した 。 [SEP]\n",
      "INFO:tensorflow:input_ids: 101 1492 1512 18378 1482 1532 23543 13639 97378 75049 15295 11651 2015 1547 3709 12563 1483 1521 97383 29644 11572 4846 1526 23543 1483 1532 26433 14440 26397 24851 23543 1483 11547 8413 1541 10845 17461 22150 24851 29512 97378 11651 11883 3901 28967 10845 11651 31150 17606 30659 22150 24851 3143 43437 11488 40167 1657 1520 29386 14811 14811 14440 97384 90973 1483 11547 62040 23543 10845 3189 2073 1527 46731 1483 1493 14166 25257 11572 97383 16181 15500 6914 1514 26397 17870 1483 6260 72521 52479 9357 1562 7753 2429 1525 26397 17870 1483 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: 1 (id = 1)\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: train-2\n",
      "INFO:tensorflow:tokens: [CLS] 赤 狐 は 扉 を ##しめ ##て 云 ひ ##ま ##した 。 僕 は ##あ ##の ##い ##つ ##か 公 爵 の 子 供 か 着 て 居 た 赤 い 上 着 な ##ら 見 た ##い ##な ##あ ##と 仔 牛 は 思 ひ ##ま ##した ##け ##れ ##とも ##も ##う 狐 か ##く ##ん ##く ##ん 向 ふ ##へ 行 く ##も ##ん ##て ##す ##から 仕 方 なく ##つ ##いて 行 き ##ま ##した 。 [SEP]\n",
      "INFO:tensorflow:input_ids: 101 8212 5608 1538 3933 1562 69106 11883 1735 1539 26397 17870 1483 2015 1538 97378 10711 14811 23393 10845 2077 5549 1537 3155 1885 1517 6043 1531 3301 1527 8212 1510 1657 6043 1533 17044 7753 1527 14811 22353 97378 11651 1780 5567 1538 3709 1539 26397 17870 23024 28028 54772 13639 23543 5608 1517 17461 18378 17461 18378 2438 1540 43458 7658 1519 13639 18378 11883 16018 17587 1781 4301 57516 23393 29512 7658 1518 26397 17870 1483 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: 1 (id = 1)\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: train-3\n",
      "INFO:tensorflow:tokens: [CLS] 俊 助 は 制 服 の 隠 し ##を ##さ ##く ##って 、 名 刺 を 辰 子 の 手 へ 渡 した 。 渡 す 時 向 う ##の 手 を 見 ると 、 青 玉 を 入 れた 金 の 指 環 か 、 細 っ ##そ ##り ##と ##その 小 指 を 繞 っていた 。 俊 助 は ##そ ##れ ##も ##また 美 しい ##と 思 った 。 [SEP]\n",
      "INFO:tensorflow:input_ids: 101 1908 2242 1538 2189 4451 1537 9182 1523 11465 44180 17461 26433 1482 2434 2193 1562 8408 3155 1537 3934 1541 5184 10568 1483 5184 1524 4363 2438 1512 10711 3934 1562 7753 43110 1482 9262 5675 1562 2071 21618 8688 1537 4037 5800 1517 1482 6577 1529 69175 15500 11651 19504 3273 4037 1562 6682 24654 1483 1908 2242 1538 69175 28028 13639 52479 6839 18658 11651 3709 12563 1483 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: 0 (id = 0)\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: train-4\n",
      "INFO:tensorflow:tokens: [CLS] 最 後 に 小 泉 孤 松 の 書 いた 「 農 家 義 人 伝 」 の 中 の 一 篇 によれは 、 平 四 郎 は 伝 吉 の 牽 いていた 馬 に 泥 田 へ 蹴 落 された ##と 云 う ##ことて ##ある 。 （ 註 三 ） [SEP]\n",
      "INFO:tensorflow:input_ids: 101 4446 3651 1534 3273 5016 3170 4507 1537 4439 17210 1492 8410 3219 6850 1763 1818 1493 1537 1683 1537 1651 6451 53840 1482 3497 2733 8575 1538 1818 2430 1537 5582 67500 9492 1534 5029 5841 1541 8285 7339 11105 11651 1735 1512 24230 17977 1483 10028 7812 1656 10029 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: 0 (id = 0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:***** Running training *****\n",
      "INFO:tensorflow:  Num examples = 9746\n",
      "INFO:tensorflow:  Batch size = 32\n",
      "INFO:tensorflow:  Num steps = 3045\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Running train on CPU\n",
      "INFO:tensorflow:*** Features ***\n",
      "INFO:tensorflow:  name = input_ids, shape = (32, 128)\n",
      "INFO:tensorflow:  name = input_mask, shape = (32, 128)\n",
      "INFO:tensorflow:  name = label_ids, shape = (32,)\n",
      "INFO:tensorflow:  name = segment_ids, shape = (32, 128)\n",
      "INFO:tensorflow:**** Trainable Variables ****\n",
      "INFO:tensorflow:  name = bert/embeddings/word_embeddings:0, shape = (105879, 768)\n",
      "INFO:tensorflow:  name = bert/embeddings/token_type_embeddings:0, shape = (2, 768)\n",
      "INFO:tensorflow:  name = bert/embeddings/position_embeddings:0, shape = (512, 768)\n",
      "INFO:tensorflow:  name = bert/embeddings/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/embeddings/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/pooler/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/pooler/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = output_weights:0, shape = (2, 768)\n",
      "INFO:tensorflow:  name = output_bias:0, shape = (2,)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "2018-11-22 08:03:10.506922: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n",
      "2018-11-22 08:03:10.588499: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:897] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2018-11-22 08:03:10.588899: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1405] Found device 0 with properties: \n",
      "name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235\n",
      "pciBusID: 0000:00:1e.0\n",
      "totalMemory: 11.17GiB freeMemory: 11.10GiB\n",
      "2018-11-22 08:03:10.588928: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1484] Adding visible gpu devices: 0\n",
      "2018-11-22 08:03:10.871943: I tensorflow/core/common_runtime/gpu/gpu_device.cc:965] Device interconnect StreamExecutor with strength 1 edge matrix:\n",
      "2018-11-22 08:03:10.871999: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971]      0 \n",
      "2018-11-22 08:03:10.872009: I tensorflow/core/common_runtime/gpu/gpu_device.cc:984] 0:   N \n",
      "2018-11-22 08:03:10.872295: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1097] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10758 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:1e.0, compute capability: 3.7)\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into aozora/cola_10_output_nockpt/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.607987\n",
      "INFO:tensorflow:examples/sec: 19.4556\n",
      "INFO:tensorflow:global_step/sec: 0.624062\n",
      "INFO:tensorflow:examples/sec: 19.97\n",
      "INFO:tensorflow:global_step/sec: 0.625288\n",
      "INFO:tensorflow:examples/sec: 20.0092\n",
      "INFO:tensorflow:global_step/sec: 0.623138\n",
      "INFO:tensorflow:examples/sec: 19.9404\n",
      "INFO:tensorflow:global_step/sec: 0.625199\n",
      "INFO:tensorflow:examples/sec: 20.0064\n",
      "INFO:tensorflow:global_step/sec: 0.623342\n",
      "INFO:tensorflow:examples/sec: 19.9469\n",
      "INFO:tensorflow:global_step/sec: 0.624894\n",
      "INFO:tensorflow:examples/sec: 19.9966\n",
      "INFO:tensorflow:global_step/sec: 0.624947\n",
      "INFO:tensorflow:examples/sec: 19.9983\n",
      "INFO:tensorflow:global_step/sec: 0.623836\n",
      "INFO:tensorflow:examples/sec: 19.9628\n",
      "INFO:tensorflow:Saving checkpoints for 1000 into aozora/cola_10_output_nockpt/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.549268\n",
      "INFO:tensorflow:examples/sec: 17.5766\n",
      "INFO:tensorflow:global_step/sec: 0.626307\n",
      "INFO:tensorflow:examples/sec: 20.0418\n",
      "INFO:tensorflow:global_step/sec: 0.623113\n",
      "INFO:tensorflow:examples/sec: 19.9396\n",
      "INFO:tensorflow:global_step/sec: 0.622471\n",
      "INFO:tensorflow:examples/sec: 19.9191\n",
      "INFO:tensorflow:global_step/sec: 0.624293\n",
      "INFO:tensorflow:examples/sec: 19.9774\n",
      "INFO:tensorflow:global_step/sec: 0.623096\n",
      "INFO:tensorflow:examples/sec: 19.9391\n",
      "INFO:tensorflow:global_step/sec: 0.623991\n",
      "INFO:tensorflow:examples/sec: 19.9677\n",
      "INFO:tensorflow:global_step/sec: 0.624331\n",
      "INFO:tensorflow:examples/sec: 19.9786\n",
      "INFO:tensorflow:global_step/sec: 0.624214\n",
      "INFO:tensorflow:examples/sec: 19.9749\n",
      "INFO:tensorflow:global_step/sec: 0.621982\n",
      "INFO:tensorflow:examples/sec: 19.9034\n",
      "INFO:tensorflow:Saving checkpoints for 2000 into aozora/cola_10_output_nockpt/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.547698\n",
      "INFO:tensorflow:examples/sec: 17.5263\n",
      "INFO:tensorflow:global_step/sec: 0.626539\n",
      "INFO:tensorflow:examples/sec: 20.0492\n",
      "INFO:tensorflow:global_step/sec: 0.623444\n",
      "INFO:tensorflow:examples/sec: 19.9502\n",
      "INFO:tensorflow:global_step/sec: 0.623539\n",
      "INFO:tensorflow:examples/sec: 19.9533\n",
      "INFO:tensorflow:global_step/sec: 0.625291\n",
      "INFO:tensorflow:examples/sec: 20.0093\n",
      "INFO:tensorflow:global_step/sec: 0.624382\n",
      "INFO:tensorflow:examples/sec: 19.9802\n",
      "INFO:tensorflow:global_step/sec: 0.626512\n",
      "INFO:tensorflow:examples/sec: 20.0484\n",
      "INFO:tensorflow:global_step/sec: 0.624061\n",
      "INFO:tensorflow:examples/sec: 19.97\n",
      "INFO:tensorflow:global_step/sec: 0.622786\n",
      "INFO:tensorflow:examples/sec: 19.9292\n",
      "INFO:tensorflow:global_step/sec: 0.622077\n",
      "INFO:tensorflow:examples/sec: 19.9065\n",
      "INFO:tensorflow:Saving checkpoints for 3000 into aozora/cola_10_output_nockpt/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 0.546727\n",
      "INFO:tensorflow:examples/sec: 17.4953\n",
      "INFO:tensorflow:Saving checkpoints for 3045 into aozora/cola_10_output_nockpt/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.0001236751.\n",
      "INFO:tensorflow:Writing example 0 of 1195\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: dev-0\n",
      "INFO:tensorflow:tokens: [CLS] （ 何 箇 月 か ##たった 後 、 僕 は 何 か ##の 話 の 次 手 に 『 悪 魔 』 の 作 家 に 彼 の 言 葉 を 話 した 。 すると ##この 作 家 は 笑 い ##なから 、 無 造 作 に 僕 に ##こ ##う 言 う ##の ##たった 。 [UNK] [UNK] 「 世 界 一 な ##らは 何 ても 好 い 。 」 ！ ） [SEP]\n",
      "INFO:tensorflow:input_ids: 101 10028 1847 6427 4448 1517 57603 3651 1482 2015 1538 1847 1517 10711 7836 1537 4827 3934 1534 1494 3767 9646 1495 1537 1853 3219 1534 3640 1537 7785 7341 1562 7836 10568 1483 20969 20864 1853 3219 1538 6381 1510 32980 1482 5466 8479 1853 1534 2015 1534 23635 23543 7785 1512 10711 57603 1483 100 100 1492 1666 5855 1651 1533 46775 1847 16432 3019 1510 1483 1493 10023 10029 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: 0 (id = 0)\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: dev-1\n",
      "INFO:tensorflow:tokens: [CLS] 林 中 の 女 の ##ふ ##く ##ろう ##か ##ま ##る ##て 口 口 に 答 え ##ま ##した 。 その 音 は 二 町 は ##かり 西 の 方 の 大 きな 藁 屋 根 の 中 に 捕 われている 穂 吉 の 処 まて 、 ほ ##ん ##のか ##す ##かに ##て ##した ##け ##れ ##とも 聞 えた ##のて ##す 。 [SEP]\n",
      "INFO:tensorflow:input_ids: 101 4516 1683 1537 3014 1537 75049 17461 60422 10845 26397 16206 11883 2404 2404 1534 6410 1514 26397 17870 1483 11547 9313 1538 1732 5848 1538 50663 7748 1537 4301 1537 2974 24725 7459 3305 4588 1537 1683 1534 4071 29304 6307 2430 1537 2145 14579 1482 1542 18378 25257 16018 86250 11883 17870 23024 28028 54772 6914 19542 27145 16018 1483 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: 1 (id = 1)\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: dev-2\n",
      "INFO:tensorflow:tokens: [CLS] 廣 い 門 の 下 には 、 この 男 の 外 に 誰 も ##ゐ ##ない 。 唯 、 所 々の 剥 けた 、 大 きな 圓 柱 に 、 蟋 蟀 か 一 匹 と ##ま ##つ ##て ##ゐ ##る 。 羅 生 門 か 、 朱 雀 大 路 にある 以 上 は 、 この 男 の 外 にも 、 雨 や ##み ##を ##する 市 女 笠 や 揉 烏 帽 子 か 、 もう 二 三 は ##あり ##さ ##う ##な ##ものてある 。 それ ##か 、 この 男 の 外 には 誰 も ##ゐ ##ない 。 [SEP]\n",
      "INFO:tensorflow:input_ids: 101 3559 1510 9028 1537 1658 11350 1482 12058 5846 1537 2965 1534 7863 1547 97388 15295 1483 2578 1482 3928 39522 2213 17052 1482 2974 24725 2763 4555 1534 1482 7626 7622 1517 1651 2308 1532 26397 23393 11883 97388 16206 1483 6833 5830 9028 1517 1482 4469 9198 2974 8252 17372 1792 1657 1538 1482 12058 5846 1537 2965 15599 1482 9221 1549 17717 11465 28402 3451 3014 6387 1549 4125 5435 3480 3155 1517 1482 89828 1732 1656 1538 29189 44180 23543 22353 35892 1483 48885 10845 1482 12058 5846 1537 2965 11350 7863 1547 97388 15295 1483 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: 0 (id = 0)\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: dev-3\n",
      "INFO:tensorflow:tokens: [CLS] 「 あ ##あ 、 そ ##う ##て ##す ##か 、 そ ##い ##し ##ゃ 私 の ##ほ ##う ##か ##や ##っ ##は ##り 詳 しく 知 って ##ます 。 この 間 まて 馬 喰 を ##や ##って ##ま ##した ##か ##ね 。 今 こ ##ろ ##は 何 をしている ##か 全 く 困 った ##も ##ん ##て ##す ##よ 。 」 [SEP]\n",
      "INFO:tensorflow:input_ids: 101 1492 1508 97378 1482 1526 23543 11883 16018 10845 1482 1526 14811 14440 97384 6260 1537 62040 23543 10845 20936 97383 11572 15500 7838 23818 6091 15188 46630 1483 12058 9038 14579 9492 2629 1562 20936 26433 26397 17870 10845 61828 1483 1773 1521 37243 11572 1847 57483 10845 2073 1519 2745 12563 13639 18378 11883 16018 46731 1483 1493 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: 1 (id = 1)\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: dev-4\n",
      "INFO:tensorflow:tokens: [CLS] 私 は 覚 束 ない 声 を 出 して 、 何 と 云 う 事 も ##なく ##こ ##う 問 い 返 し ##ま ##した 。 実 際 その 時 の 私 には 、 また 摩 利 信 乃 法 師 を 殺 そ ##う ##とも 、 殺 す ##ま ##い ##とも 、 は ##っ ##き ##り ##した 決 断 かつ ##か ##すに ##いた ##のて ##こ ##さ ##い ##ます 。 か 、 そ ##う 云 う 内 にも 私 の 甥 か 、 今 度 は ##ふ ##り 向 く ##ら ##しい 容 子 も ##なく 、 し ##っと ##その 小 屋 を 見 守 り ##なから 、 [SEP]\n",
      "INFO:tensorflow:input_ids: 101 6260 1538 7758 4490 35141 2949 1562 2157 11656 1482 1847 1532 1735 1512 1731 1547 26557 23635 23543 2589 1510 8428 1523 26397 17870 1483 3204 9180 11547 4363 1537 6260 11350 1482 12141 4173 2183 1920 1696 5020 3470 1562 4885 1526 23543 54772 1482 4885 1524 26397 14811 54772 1482 1538 97383 16181 15500 17870 4975 4297 59750 10845 93133 23876 27145 23635 44180 14811 46630 1483 1517 1482 1526 23543 1735 1512 2093 15599 6260 1537 5834 1517 1482 1773 3530 1538 75049 15500 2438 1519 17044 37237 3221 3155 1547 26557 1482 1523 69416 19504 3273 3305 1562 7753 3188 1555 32980 1482 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label: 0 (id = 0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:***** Running evaluation *****\n",
      "INFO:tensorflow:  Num examples = 1195\n",
      "INFO:tensorflow:  Batch size = 8\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Running eval on CPU\n",
      "INFO:tensorflow:*** Features ***\n",
      "INFO:tensorflow:  name = input_ids, shape = (?, 128)\n",
      "INFO:tensorflow:  name = input_mask, shape = (?, 128)\n",
      "INFO:tensorflow:  name = label_ids, shape = (?,)\n",
      "INFO:tensorflow:  name = segment_ids, shape = (?, 128)\n",
      "INFO:tensorflow:**** Trainable Variables ****\n",
      "INFO:tensorflow:  name = bert/embeddings/word_embeddings:0, shape = (105879, 768)\n",
      "INFO:tensorflow:  name = bert/embeddings/token_type_embeddings:0, shape = (2, 768)\n",
      "INFO:tensorflow:  name = bert/embeddings/position_embeddings:0, shape = (512, 768)\n",
      "INFO:tensorflow:  name = bert/embeddings/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/embeddings/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/pooler/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/pooler/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = output_weights:0, shape = (2, 768)\n",
      "INFO:tensorflow:  name = output_bias:0, shape = (2,)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2018-11-22-09:26:48\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "2018-11-22 09:26:49.480428: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1484] Adding visible gpu devices: 0\n",
      "2018-11-22 09:26:49.480489: I tensorflow/core/common_runtime/gpu/gpu_device.cc:965] Device interconnect StreamExecutor with strength 1 edge matrix:\n",
      "2018-11-22 09:26:49.480505: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971]      0 \n",
      "2018-11-22 09:26:49.480517: I tensorflow/core/common_runtime/gpu/gpu_device.cc:984] 0:   N \n",
      "2018-11-22 09:26:49.480639: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1097] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10758 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:1e.0, compute capability: 3.7)\n",
      "INFO:tensorflow:Restoring parameters from aozora/cola_10_output_nockpt/model.ckpt-3045\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2018-11-22-09:27:21\n",
      "INFO:tensorflow:Saving dict for global step 3045: eval_accuracy = 0.98493725, eval_loss = 0.09885448, global_step = 3045, loss = 0.09844306\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 3045: aozora/cola_10_output_nockpt/model.ckpt-3045\n",
      "INFO:tensorflow:***** Eval results *****\n",
      "INFO:tensorflow:  eval_accuracy = 0.98493725\n",
      "INFO:tensorflow:  eval_loss = 0.09885448\n",
      "INFO:tensorflow:  global_step = 3045\n",
      "INFO:tensorflow:  loss = 0.09844306\n"
     ]
    }
   ],
   "source": [
    "# epoch 10 without checkpoint\n",
    "! ./aozora_cola.sh -e 10 -p /opt/p3/bin/python -n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}