seq2seqOT.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "seq2seqOT.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/denyHell/seq2seqOT/blob/master/seq2seqOT.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "1j9ZB7WTqQsL",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        },
        "outputId": "0cbb762d-4fc3-4614-9003-597c5d2f1dbd"
      },
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/vyraun/assignment_2.git"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'assignment_2'...\n",
            "remote: Enumerating objects: 480, done.\u001b[K\n",
            "remote: Total 480 (delta 0), reused 0 (delta 0), pack-reused 480\u001b[K\n",
            "Receiving objects: 100% (480/480), 207.14 MiB | 31.60 MiB/s, done.\n",
            "Resolving deltas: 100% (278/278), done.\n",
            "Checking out files: 100% (155/155), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "HSC5CINOqTNy",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import os\n",
        "os.chdir(\"/content/assignment_2/scripts/\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Y2xczXx6ssRC",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        },
        "outputId": "d84baefe-dd09-455a-847e-18e84c75feef"
      },
      "cell_type": "code",
      "source": [
        "!bash build_vocab.sh en-be"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "be\n",
            "en\n",
            "read in source sentences: ../data/train.en-be.be.txt\n",
            "read in target sentences: ../data/train.en-be.en.txt\n",
            "initialize source vocabulary ..\n",
            "number of word types: 16784, number of word types w/ frequency >= 2: 5908\n",
            "initialize target vocabulary ..\n",
            "number of word types: 9562, number of word types w/ frequency >= 2: 4829\n",
            "generated vocabulary, source 5912 words, target 4831 words\n",
            "vocabulary saved to ../data/vocab_en-be.bin\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "2t3k70sAxZZs",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1262
        },
        "outputId": "9adb3b8b-291d-40c0-86b6-0e4bc9480f31"
      },
      "cell_type": "code",
      "source": [
        "!pip install geomloss[full]"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting geomloss[full]\n",
            "  Downloading https://files.pythonhosted.org/packages/04/e3/e6feb0996aa00427da17a257fddf85f695ae02c0af43b0ddb2f66d647fcd/geomloss-0.2.1-py3-none-any.whl\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from geomloss[full]) (1.16.2)\n",
            "Collecting pykeops[full]; extra == \"full\" (from geomloss[full])\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/fe/70/e7b86027dd9460e852823815062e74b71b0eb5488aaa229f9517b09d8ba0/pykeops-1.0.1.tar.gz (236kB)\n",
            "\u001b[K    100% |████████████████████████████████| 245kB 7.7MB/s \n",
            "\u001b[?25hCollecting GPUtil (from pykeops[full]; extra == \"full\"->geomloss[full])\n",
            "  Downloading https://files.pythonhosted.org/packages/ed/0e/5c61eedde9f6c87713e89d794f01e378cfd9565847d4576fa627d758c554/GPUtil-1.4.0.tar.gz\n",
            "Requirement already satisfied: sphinx in /usr/local/lib/python3.6/dist-packages (from pykeops[full]; extra == \"full\"->geomloss[full]) (1.8.5)\n",
            "Collecting sphinx-gallery (from pykeops[full]; extra == \"full\"->geomloss[full])\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/1e/bf/91fef7937e91d866ef14a1b2a72d6b32942a27bd0dbc0109d524d7d742dd/sphinx-gallery-0.3.1.tar.gz (88kB)\n",
            "\u001b[K    100% |████████████████████████████████| 92kB 31.8MB/s \n",
            "\u001b[?25hCollecting recommonmark (from pykeops[full]; extra == \"full\"->geomloss[full])\n",
            "  Downloading https://files.pythonhosted.org/packages/9b/3d/92ea48401622510e57b4bdaa74dc9db2fb9e9e892324b48f9c02d716a93a/recommonmark-0.5.0-py2.py3-none-any.whl\n",
            "Collecting sphinxcontrib-httpdomain (from pykeops[full]; extra == \"full\"->geomloss[full])\n",
            "  Downloading https://files.pythonhosted.org/packages/79/35/6f43bde0c4ead866c349a1fa0ff732a31613b3f462ad9b9630c03a958f41/sphinxcontrib_httpdomain-1.7.0-py2.py3-none-any.whl\n",
            "Collecting sphinx_rtd_theme (from pykeops[full]; extra == \"full\"->geomloss[full])\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/60/b4/4df37087a1d36755e3a3bfd2a30263f358d2dea21938240fa02313d45f51/sphinx_rtd_theme-0.4.3-py2.py3-none-any.whl (6.4MB)\n",
            "\u001b[K    100% |████████████████████████████████| 6.4MB 7.9MB/s \n",
            "\u001b[?25hCollecting breathe (from pykeops[full]; extra == \"full\"->geomloss[full])\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/6c/af/8c43c9a50ab9b574b84844e2145ea7c2c3b0b8f2226a5d31517a6a1c19cc/breathe-4.12.0-py2.py3-none-any.whl (74kB)\n",
            "\u001b[K    100% |████████████████████████████████| 81kB 29.6MB/s \n",
            "\u001b[?25hCollecting gcc7 (from pykeops[full]; extra == \"full\"->geomloss[full])\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/42/99/e0758ee08b73c728432381e1fccd445a8f8867a190ff5df5749d8c909922/gcc7-0.0.7-py2.py3-none-manylinux1_x86_64.whl (52.2MB)\n",
            "\u001b[K    100% |████████████████████████████████| 52.2MB 613kB/s \n",
            "\u001b[?25hRequirement already satisfied: cmake in /usr/local/lib/python3.6/dist-packages (from pykeops[full]; extra == \"full\"->geomloss[full]) (3.12.0)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (from pykeops[full]; extra == \"full\"->geomloss[full]) (3.0.3)\n",
            "Requirement already satisfied: imageio in /usr/local/lib/python3.6/dist-packages (from pykeops[full]; extra == \"full\"->geomloss[full]) (2.4.1)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.6/dist-packages (from pykeops[full]; extra == \"full\"->geomloss[full]) (1.0.1.post2)\n",
            "Requirement already satisfied: Pygments>=2.0 in /usr/local/lib/python3.6/dist-packages (from sphinx->pykeops[full]; extra == \"full\"->geomloss[full]) (2.1.3)\n",
            "Requirement already satisfied: snowballstemmer>=1.1 in /usr/local/lib/python3.6/dist-packages (from sphinx->pykeops[full]; extra == \"full\"->geomloss[full]) (1.2.1)\n",
            "Requirement already satisfied: imagesize in /usr/local/lib/python3.6/dist-packages (from sphinx->pykeops[full]; extra == \"full\"->geomloss[full]) (1.1.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from sphinx->pykeops[full]; extra == \"full\"->geomloss[full]) (40.9.0)\n",
            "Requirement already satisfied: requests>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from sphinx->pykeops[full]; extra == \"full\"->geomloss[full]) (2.18.4)\n",
            "Requirement already satisfied: docutils>=0.11 in /usr/local/lib/python3.6/dist-packages (from sphinx->pykeops[full]; extra == \"full\"->geomloss[full]) (0.14)\n",
            "Requirement already satisfied: alabaster<0.8,>=0.7 in /usr/local/lib/python3.6/dist-packages (from sphinx->pykeops[full]; extra == \"full\"->geomloss[full]) (0.7.12)\n",
            "Requirement already satisfied: Jinja2>=2.3 in /usr/local/lib/python3.6/dist-packages (from sphinx->pykeops[full]; extra == \"full\"->geomloss[full]) (2.10.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from sphinx->pykeops[full]; extra == \"full\"->geomloss[full]) (1.11.0)\n",
            "Requirement already satisfied: babel!=2.0,>=1.3 in /usr/local/lib/python3.6/dist-packages (from sphinx->pykeops[full]; extra == \"full\"->geomloss[full]) (2.6.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from sphinx->pykeops[full]; extra == \"full\"->geomloss[full]) (19.0)\n",
            "Requirement already satisfied: sphinxcontrib-websupport in /usr/local/lib/python3.6/dist-packages (from sphinx->pykeops[full]; extra == \"full\"->geomloss[full]) (1.1.0)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.6/dist-packages (from sphinx-gallery->pykeops[full]; extra == \"full\"->geomloss[full]) (4.3.0)\n",
            "Collecting commonmark>=0.7.3 (from recommonmark->pykeops[full]; extra == \"full\"->geomloss[full])\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ab/ca/439c88039583a29564a0043186875258e9a4f041fb5c422cd387b8e10175/commonmark-0.8.1-py2.py3-none-any.whl (47kB)\n",
            "\u001b[K    100% |████████████████████████████████| 51kB 23.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->pykeops[full]; extra == \"full\"->geomloss[full]) (2.4.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->pykeops[full]; extra == \"full\"->geomloss[full]) (2.5.3)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->pykeops[full]; extra == \"full\"->geomloss[full]) (1.0.1)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib->pykeops[full]; extra == \"full\"->geomloss[full]) (0.10.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0.0->sphinx->pykeops[full]; extra == \"full\"->geomloss[full]) (2019.3.9)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0.0->sphinx->pykeops[full]; extra == \"full\"->geomloss[full]) (3.0.4)\n",
            "Requirement already satisfied: urllib3<1.23,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0.0->sphinx->pykeops[full]; extra == \"full\"->geomloss[full]) (1.22)\n",
            "Requirement already satisfied: idna<2.7,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0.0->sphinx->pykeops[full]; extra == \"full\"->geomloss[full]) (2.6)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.6/dist-packages (from Jinja2>=2.3->sphinx->pykeops[full]; extra == \"full\"->geomloss[full]) (1.1.1)\n",
            "Requirement already satisfied: pytz>=0a in /usr/local/lib/python3.6/dist-packages (from babel!=2.0,>=1.3->sphinx->pykeops[full]; extra == \"full\"->geomloss[full]) (2018.9)\n",
            "Requirement already satisfied: olefile in /usr/local/lib/python3.6/dist-packages (from pillow->sphinx-gallery->pykeops[full]; extra == \"full\"->geomloss[full]) (0.46)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from commonmark>=0.7.3->recommonmark->pykeops[full]; extra == \"full\"->geomloss[full]) (0.16.0)\n",
            "Building wheels for collected packages: pykeops, GPUtil, sphinx-gallery\n",
            "  Building wheel for pykeops (setup.py) ... \u001b[?25ldone\n",
            "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/87/2b/2c/5c07e0b3fd5941d9d93410d3d72d683798ae05564a642fba9d\n",
            "  Building wheel for GPUtil (setup.py) ... \u001b[?25ldone\n",
            "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/3d/77/07/80562de4bb0786e5ea186911a2c831fdd0018bda69beab71fd\n",
            "  Building wheel for sphinx-gallery (setup.py) ... \u001b[?25ldone\n",
            "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/fa/6d/ab/c2893a6bf6a2f9537a06302ede3bbdd0105deab1d4cc4e6682\n",
            "Successfully built pykeops GPUtil sphinx-gallery\n",
            "Installing collected packages: GPUtil, sphinx-gallery, commonmark, recommonmark, sphinxcontrib-httpdomain, sphinx-rtd-theme, breathe, gcc7, pykeops, geomloss\n",
            "Successfully installed GPUtil-1.4.0 breathe-4.12.0 commonmark-0.8.1 gcc7-0.0.7 geomloss-0.2.1 pykeops-1.0.1 recommonmark-0.5.0 sphinx-gallery-0.3.1 sphinx-rtd-theme-0.4.3 sphinxcontrib-httpdomain-1.7.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "sphinxcontrib"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "3zkrO0QvyvEH",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 39069
        },
        "outputId": "57d08ac8-39a8-47d4-dc0a-77009bbbba06"
      },
      "cell_type": "code",
      "source": [
        "!bash train1.sh en-az\n"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "source az\n",
            "target en\n",
            "save re\n",
            "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/rnn.py:46: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.2 and num_layers=1\n",
            "  \"num_layers={}\".format(dropout, num_layers))\n",
            "../models/nmt.py:176: UserWarning: nn.init.uniform is now deprecated in favor of nn.init.uniform_.\n",
            "  torch.nn.init.uniform(param, a=-uniform_init, b=uniform_init)\n",
            "/usr/local/lib/python3.6/dist-packages/torch/nn/_reduction.py:49: UserWarning: size_average and reduce args will be deprecated, please use reduction='none' instead.\n",
            "  warnings.warn(warning.format(ret))\n",
            "begin Maximum Likelihood training\n",
            "/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py:1320: UserWarning: nn.functional.tanh is deprecated. Use torch.tanh instead.\n",
            "  warnings.warn(\"nn.functional.tanh is deprecated. Use torch.tanh instead.\")\n",
            "../models/nmt.py:628: UserWarning: torch.nn.utils.clip_grad_norm is now deprecated in favor of torch.nn.utils.clip_grad_norm_.\n",
            "  torch.nn.utils.clip_grad_norm(list(model.encoder.parameters()) + list(model.decoder.parameters()), clip_grad)\n",
            "epoch 1, iter 10, avg. loss 151.95, avg. ppl 1280.04 cum. examples 160, speed 1128.04 words/sec, time elapsed 3.01 sec\n",
            "epoch 1, iter 20, avg. loss 146.21, avg. ppl 465.65 cum. examples 320, speed 1129.96 words/sec, time elapsed 6.38 sec\n",
            "epoch 1, iter 30, avg. loss 120.68, avg. ppl 347.62 cum. examples 480, speed 1618.75 words/sec, time elapsed 8.42 sec\n",
            "epoch 1, iter 40, avg. loss 115.78, avg. ppl 327.83 cum. examples 640, speed 1313.51 words/sec, time elapsed 10.86 sec\n",
            "epoch 1, iter 50, avg. loss 134.40, avg. ppl 333.25 cum. examples 800, speed 1465.33 words/sec, time elapsed 13.38 sec\n",
            "epoch 1, iter 60, avg. loss 134.30, avg. ppl 296.96 cum. examples 960, speed 1176.63 words/sec, time elapsed 16.59 sec\n",
            "epoch 1, iter 70, avg. loss 120.57, avg. ppl 257.20 cum. examples 1120, speed 1409.70 words/sec, time elapsed 19.06 sec\n",
            "epoch 1, iter 80, avg. loss 137.33, avg. ppl 265.37 cum. examples 1280, speed 1266.49 words/sec, time elapsed 22.16 sec\n",
            "epoch 1, iter 90, avg. loss 123.71, avg. ppl 238.72 cum. examples 1440, speed 1205.50 words/sec, time elapsed 25.16 sec\n",
            "epoch 1, iter 100, avg. loss 114.75, avg. ppl 211.20 cum. examples 1600, speed 1580.52 words/sec, time elapsed 27.33 sec\n",
            "epoch 1, iter 110, avg. loss 114.12, avg. ppl 204.12 cum. examples 1760, speed 1355.83 words/sec, time elapsed 29.87 sec\n",
            "epoch 1, iter 120, avg. loss 110.08, avg. ppl 186.39 cum. examples 1920, speed 1625.01 words/sec, time elapsed 31.94 sec\n",
            "epoch 1, iter 130, avg. loss 125.94, avg. ppl 208.91 cum. examples 2080, speed 1345.48 words/sec, time elapsed 34.74 sec\n",
            "epoch 1, iter 140, avg. loss 121.08, avg. ppl 181.15 cum. examples 2240, speed 1304.79 words/sec, time elapsed 37.60 sec\n",
            "epoch 1, iter 150, avg. loss 106.11, avg. ppl 157.21 cum. examples 2400, speed 1316.23 words/sec, time elapsed 40.15 sec\n",
            "epoch 1, iter 160, avg. loss 116.25, avg. ppl 162.00 cum. examples 2560, speed 1288.76 words/sec, time elapsed 42.99 sec\n",
            "epoch 1, iter 170, avg. loss 108.19, avg. ppl 149.69 cum. examples 2720, speed 1466.60 words/sec, time elapsed 45.34 sec\n",
            "epoch 1, iter 180, avg. loss 109.14, avg. ppl 140.76 cum. examples 2880, speed 1424.67 words/sec, time elapsed 47.82 sec\n",
            "epoch 1, iter 190, avg. loss 106.64, avg. ppl 143.41 cum. examples 3040, speed 1418.79 words/sec, time elapsed 50.24 sec\n",
            "epoch 1, iter 200, avg. loss 96.57, avg. ppl 136.47 cum. examples 3200, speed 1590.53 words/sec, time elapsed 52.22 sec\n",
            "epoch 1, iter 210, avg. loss 116.19, avg. ppl 142.45 cum. examples 3360, speed 1415.78 words/sec, time elapsed 54.87 sec\n",
            "epoch 1, iter 220, avg. loss 107.81, avg. ppl 130.66 cum. examples 3520, speed 1440.38 words/sec, time elapsed 57.32 sec\n",
            "epoch 1, iter 230, avg. loss 106.39, avg. ppl 124.23 cum. examples 3680, speed 1436.23 words/sec, time elapsed 59.78 sec\n",
            "epoch 1, iter 240, avg. loss 109.98, avg. ppl 129.35 cum. examples 3840, speed 1445.40 words/sec, time elapsed 62.29 sec\n",
            "epoch 1, iter 250, avg. loss 101.65, avg. ppl 118.02 cum. examples 4000, speed 1307.40 words/sec, time elapsed 64.89 sec\n",
            "epoch 1, iter 260, avg. loss 101.24, avg. ppl 138.12 cum. examples 4160, speed 1507.74 words/sec, time elapsed 67.07 sec\n",
            "epoch 1, iter 270, avg. loss 113.89, avg. ppl 121.38 cum. examples 4320, speed 1293.73 words/sec, time elapsed 70.01 sec\n",
            "epoch 1, iter 280, avg. loss 108.32, avg. ppl 131.90 cum. examples 4480, speed 1374.32 words/sec, time elapsed 72.59 sec\n",
            "epoch 1, iter 290, avg. loss 101.73, avg. ppl 110.27 cum. examples 4640, speed 1360.67 words/sec, time elapsed 75.13 sec\n",
            "epoch 1, iter 300, avg. loss 122.93, avg. ppl 120.74 cum. examples 4800, speed 1259.91 words/sec, time elapsed 78.39 sec\n",
            "epoch 1, iter 310, avg. loss 105.87, avg. ppl 113.15 cum. examples 4960, speed 1378.29 words/sec, time elapsed 80.99 sec\n",
            "epoch 1, iter 320, avg. loss 106.03, avg. ppl 127.72 cum. examples 5120, speed 1415.90 words/sec, time elapsed 83.46 sec\n",
            "epoch 1, iter 330, avg. loss 92.09, avg. ppl 95.74 cum. examples 5280, speed 1364.52 words/sec, time elapsed 85.83 sec\n",
            "epoch 1, iter 340, avg. loss 90.18, avg. ppl 100.31 cum. examples 5440, speed 1600.21 words/sec, time elapsed 87.78 sec\n",
            "epoch 1, iter 350, avg. loss 103.07, avg. ppl 112.76 cum. examples 5600, speed 1505.67 words/sec, time elapsed 90.10 sec\n",
            "epoch 1, iter 360, avg. loss 112.43, avg. ppl 110.79 cum. examples 5760, speed 1399.16 words/sec, time elapsed 92.83 sec\n",
            "epoch 1, iter 370, avg. loss 98.38, avg. ppl 101.93 cum. examples 5920, speed 1493.50 words/sec, time elapsed 95.11 sec\n",
            "epoch 1, iter 380, avg. loss 95.50, avg. ppl 100.99 cum. examples 6080, speed 1361.01 words/sec, time elapsed 97.55 sec\n",
            "epoch 1, iter 390, avg. loss 105.55, avg. ppl 109.25 cum. examples 6240, speed 1293.16 words/sec, time elapsed 100.33 sec\n",
            "epoch 1, iter 400, avg. loss 106.90, avg. ppl 106.75 cum. examples 6400, speed 1381.04 words/sec, time elapsed 102.98 sec\n",
            "epoch 2, iter 410, avg. loss 98.71, avg. ppl 77.90 cum. examples 6558, speed 1213.71 words/sec, time elapsed 105.93 sec\n",
            "epoch 2, iter 420, avg. loss 97.57, avg. ppl 75.89 cum. examples 6718, speed 1440.51 words/sec, time elapsed 108.43 sec\n",
            "epoch 2, iter 430, avg. loss 89.87, avg. ppl 72.10 cum. examples 6878, speed 1398.76 words/sec, time elapsed 110.84 sec\n",
            "epoch 2, iter 440, avg. loss 94.71, avg. ppl 72.46 cum. examples 7038, speed 1313.65 words/sec, time elapsed 113.53 sec\n",
            "epoch 2, iter 450, avg. loss 93.93, avg. ppl 73.53 cum. examples 7198, speed 1363.95 words/sec, time elapsed 116.09 sec\n",
            "epoch 2, iter 460, avg. loss 95.36, avg. ppl 78.90 cum. examples 7358, speed 1194.25 words/sec, time elapsed 119.02 sec\n",
            "epoch 2, iter 470, avg. loss 101.23, avg. ppl 80.22 cum. examples 7518, speed 1448.04 words/sec, time elapsed 121.57 sec\n",
            "epoch 2, iter 480, avg. loss 90.02, avg. ppl 74.13 cum. examples 7678, speed 1350.64 words/sec, time elapsed 124.05 sec\n",
            "epoch 2, iter 490, avg. loss 91.09, avg. ppl 77.03 cum. examples 7838, speed 1543.51 words/sec, time elapsed 126.22 sec\n",
            "epoch 2, iter 500, avg. loss 103.91, avg. ppl 70.03 cum. examples 7998, speed 1319.46 words/sec, time elapsed 129.19 sec\n",
            "epoch 2, iter 510, avg. loss 95.15, avg. ppl 76.12 cum. examples 8158, speed 1354.47 words/sec, time elapsed 131.78 sec\n",
            "epoch 2, iter 520, avg. loss 103.64, avg. ppl 80.86 cum. examples 8318, speed 1367.72 words/sec, time elapsed 134.54 sec\n",
            "epoch 2, iter 530, avg. loss 86.13, avg. ppl 69.53 cum. examples 8478, speed 1487.75 words/sec, time elapsed 136.72 sec\n",
            "epoch 2, iter 540, avg. loss 88.63, avg. ppl 68.67 cum. examples 8638, speed 1301.45 words/sec, time elapsed 139.30 sec\n",
            "epoch 2, iter 550, avg. loss 90.82, avg. ppl 70.12 cum. examples 8798, speed 1378.12 words/sec, time elapsed 141.78 sec\n",
            "epoch 2, iter 560, avg. loss 106.10, avg. ppl 76.83 cum. examples 8958, speed 1377.92 words/sec, time elapsed 144.62 sec\n",
            "epoch 2, iter 570, avg. loss 88.99, avg. ppl 71.02 cum. examples 9118, speed 1414.57 words/sec, time elapsed 146.98 sec\n",
            "epoch 2, iter 580, avg. loss 82.50, avg. ppl 74.63 cum. examples 9278, speed 1405.56 words/sec, time elapsed 149.16 sec\n",
            "epoch 2, iter 590, avg. loss 92.20, avg. ppl 73.94 cum. examples 9438, speed 1155.72 words/sec, time elapsed 152.12 sec\n",
            "epoch 2, iter 600, avg. loss 84.22, avg. ppl 70.74 cum. examples 9598, speed 1446.56 words/sec, time elapsed 154.31 sec\n",
            "epoch 2, iter 610, avg. loss 87.41, avg. ppl 64.23 cum. examples 9758, speed 1450.52 words/sec, time elapsed 156.63 sec\n",
            "epoch 2, iter 620, avg. loss 99.80, avg. ppl 70.67 cum. examples 9918, speed 1440.58 words/sec, time elapsed 159.23 sec\n",
            "epoch 2, iter 630, avg. loss 96.40, avg. ppl 77.54 cum. examples 10078, speed 1296.46 words/sec, time elapsed 161.97 sec\n",
            "epoch 2, iter 640, avg. loss 89.80, avg. ppl 67.01 cum. examples 10238, speed 1419.31 words/sec, time elapsed 164.37 sec\n",
            "epoch 2, iter 650, avg. loss 91.64, avg. ppl 66.13 cum. examples 10398, speed 1485.69 words/sec, time elapsed 166.73 sec\n",
            "epoch 2, iter 660, avg. loss 97.32, avg. ppl 69.62 cum. examples 10558, speed 1474.73 words/sec, time elapsed 169.22 sec\n",
            "epoch 2, iter 670, avg. loss 104.06, avg. ppl 74.27 cum. examples 10718, speed 1278.65 words/sec, time elapsed 172.24 sec\n",
            "epoch 2, iter 680, avg. loss 95.73, avg. ppl 70.09 cum. examples 10878, speed 1402.48 words/sec, time elapsed 174.81 sec\n",
            "epoch 2, iter 690, avg. loss 89.93, avg. ppl 67.26 cum. examples 11038, speed 1078.38 words/sec, time elapsed 177.98 sec\n",
            "epoch 2, iter 700, avg. loss 108.37, avg. ppl 73.04 cum. examples 11198, speed 1140.25 words/sec, time elapsed 181.52 sec\n",
            "epoch 2, iter 710, avg. loss 101.13, avg. ppl 70.13 cum. examples 11358, speed 1399.52 words/sec, time elapsed 184.24 sec\n",
            "epoch 2, iter 720, avg. loss 97.11, avg. ppl 68.19 cum. examples 11518, speed 1132.94 words/sec, time elapsed 187.49 sec\n",
            "epoch 2, iter 730, avg. loss 97.48, avg. ppl 69.37 cum. examples 11678, speed 1296.54 words/sec, time elapsed 190.33 sec\n",
            "epoch 2, iter 740, avg. loss 95.28, avg. ppl 68.40 cum. examples 11838, speed 1251.99 words/sec, time elapsed 193.21 sec\n",
            "epoch 2, iter 750, avg. loss 91.08, avg. ppl 68.64 cum. examples 11998, speed 1225.44 words/sec, time elapsed 196.02 sec\n",
            "epoch 2, iter 760, avg. loss 94.14, avg. ppl 72.87 cum. examples 12158, speed 1471.68 words/sec, time elapsed 198.41 sec\n",
            "epoch 2, iter 770, avg. loss 93.45, avg. ppl 70.38 cum. examples 12318, speed 1325.97 words/sec, time elapsed 201.06 sec\n",
            "epoch 2, iter 780, avg. loss 97.13, avg. ppl 70.31 cum. examples 12478, speed 1150.48 words/sec, time elapsed 204.24 sec\n",
            "epoch 2, iter 790, avg. loss 88.57, avg. ppl 66.60 cum. examples 12638, speed 1435.06 words/sec, time elapsed 206.59 sec\n",
            "epoch 2, iter 800, avg. loss 94.59, avg. ppl 70.60 cum. examples 12798, speed 1220.54 words/sec, time elapsed 209.50 sec\n",
            "epoch 2, iter 800, cum. loss 103.81, cum. ppl 109.58 cum. examples 12798\n",
            "begin validation ...\n",
            "validation: iter 800, dev. ppl 57.212130\n",
            "save currently the best model to [results/en-az/model.bin]\n",
            "epoch 3, iter 810, avg. loss 76.02, avg. ppl 42.78 cum. examples 158, speed 397.81 words/sec, time elapsed 217.54 sec\n",
            "epoch 3, iter 820, avg. loss 81.56, avg. ppl 37.04 cum. examples 318, speed 1528.20 words/sec, time elapsed 219.90 sec\n",
            "epoch 3, iter 830, avg. loss 79.12, avg. ppl 38.01 cum. examples 478, speed 1295.77 words/sec, time elapsed 222.59 sec\n",
            "epoch 3, iter 840, avg. loss 77.00, avg. ppl 38.53 cum. examples 638, speed 1540.05 words/sec, time elapsed 224.78 sec\n",
            "epoch 3, iter 850, avg. loss 76.02, avg. ppl 36.74 cum. examples 798, speed 1295.59 words/sec, time elapsed 227.39 sec\n",
            "epoch 3, iter 860, avg. loss 79.30, avg. ppl 38.52 cum. examples 958, speed 1621.60 words/sec, time elapsed 229.53 sec\n",
            "epoch 3, iter 870, avg. loss 87.44, avg. ppl 46.63 cum. examples 1118, speed 1330.87 words/sec, time elapsed 232.27 sec\n",
            "epoch 3, iter 880, avg. loss 83.53, avg. ppl 41.55 cum. examples 1278, speed 1194.51 words/sec, time elapsed 235.27 sec\n",
            "epoch 3, iter 890, avg. loss 79.74, avg. ppl 39.30 cum. examples 1438, speed 1538.92 words/sec, time elapsed 237.53 sec\n",
            "epoch 3, iter 900, avg. loss 83.60, avg. ppl 40.58 cum. examples 1598, speed 1529.58 words/sec, time elapsed 239.89 sec\n",
            "epoch 3, iter 910, avg. loss 83.41, avg. ppl 43.43 cum. examples 1758, speed 1363.29 words/sec, time elapsed 242.48 sec\n",
            "epoch 3, iter 920, avg. loss 86.33, avg. ppl 40.57 cum. examples 1918, speed 1511.83 words/sec, time elapsed 244.95 sec\n",
            "epoch 3, iter 930, avg. loss 88.81, avg. ppl 46.06 cum. examples 2078, speed 1340.47 words/sec, time elapsed 247.72 sec\n",
            "epoch 3, iter 940, avg. loss 87.13, avg. ppl 43.77 cum. examples 2238, speed 1238.41 words/sec, time elapsed 250.70 sec\n",
            "epoch 3, iter 950, avg. loss 84.35, avg. ppl 41.01 cum. examples 2398, speed 1380.33 words/sec, time elapsed 253.33 sec\n",
            "epoch 3, iter 960, avg. loss 72.71, avg. ppl 37.49 cum. examples 2558, speed 1651.33 words/sec, time elapsed 255.27 sec\n",
            "epoch 3, iter 970, avg. loss 78.85, avg. ppl 40.39 cum. examples 2718, speed 1313.01 words/sec, time elapsed 257.87 sec\n",
            "epoch 3, iter 980, avg. loss 71.14, avg. ppl 37.00 cum. examples 2878, speed 1272.28 words/sec, time elapsed 260.35 sec\n",
            "epoch 3, iter 990, avg. loss 77.09, avg. ppl 40.42 cum. examples 3038, speed 1230.27 words/sec, time elapsed 263.06 sec\n",
            "epoch 3, iter 1000, avg. loss 82.57, avg. ppl 41.46 cum. examples 3198, speed 1320.39 words/sec, time elapsed 265.75 sec\n",
            "epoch 3, iter 1010, avg. loss 85.95, avg. ppl 47.13 cum. examples 3358, speed 1223.65 words/sec, time elapsed 268.66 sec\n",
            "epoch 3, iter 1020, avg. loss 84.13, avg. ppl 45.05 cum. examples 3518, speed 1400.40 words/sec, time elapsed 271.19 sec\n",
            "epoch 3, iter 1030, avg. loss 89.87, avg. ppl 47.42 cum. examples 3678, speed 997.49 words/sec, time elapsed 274.92 sec\n",
            "epoch 3, iter 1040, avg. loss 80.85, avg. ppl 40.42 cum. examples 3838, speed 1384.33 words/sec, time elapsed 277.45 sec\n",
            "epoch 3, iter 1050, avg. loss 76.99, avg. ppl 39.98 cum. examples 3998, speed 1343.91 words/sec, time elapsed 279.93 sec\n",
            "epoch 3, iter 1060, avg. loss 82.87, avg. ppl 47.15 cum. examples 4158, speed 1407.65 words/sec, time elapsed 282.38 sec\n",
            "epoch 3, iter 1070, avg. loss 93.46, avg. ppl 48.24 cum. examples 4318, speed 1487.66 words/sec, time elapsed 284.97 sec\n",
            "epoch 3, iter 1080, avg. loss 78.37, avg. ppl 40.06 cum. examples 4478, speed 1580.62 words/sec, time elapsed 287.12 sec\n",
            "epoch 3, iter 1090, avg. loss 86.24, avg. ppl 41.48 cum. examples 4638, speed 1340.30 words/sec, time elapsed 289.88 sec\n",
            "epoch 3, iter 1100, avg. loss 78.59, avg. ppl 40.42 cum. examples 4798, speed 1188.13 words/sec, time elapsed 292.75 sec\n",
            "epoch 3, iter 1110, avg. loss 88.58, avg. ppl 46.23 cum. examples 4958, speed 1670.11 words/sec, time elapsed 294.96 sec\n",
            "epoch 3, iter 1120, avg. loss 92.53, avg. ppl 46.92 cum. examples 5118, speed 1355.56 words/sec, time elapsed 297.80 sec\n",
            "epoch 3, iter 1130, avg. loss 90.44, avg. ppl 44.22 cum. examples 5278, speed 1359.57 words/sec, time elapsed 300.61 sec\n",
            "epoch 3, iter 1140, avg. loss 89.89, avg. ppl 43.34 cum. examples 5438, speed 1176.83 words/sec, time elapsed 303.85 sec\n",
            "epoch 3, iter 1150, avg. loss 82.93, avg. ppl 44.73 cum. examples 5598, speed 1481.54 words/sec, time elapsed 306.21 sec\n",
            "epoch 3, iter 1160, avg. loss 79.07, avg. ppl 38.16 cum. examples 5758, speed 1232.92 words/sec, time elapsed 309.02 sec\n",
            "epoch 3, iter 1170, avg. loss 74.70, avg. ppl 36.57 cum. examples 5918, speed 1469.81 words/sec, time elapsed 311.28 sec\n",
            "epoch 3, iter 1180, avg. loss 91.44, avg. ppl 46.72 cum. examples 6078, speed 1253.87 words/sec, time elapsed 314.32 sec\n",
            "epoch 3, iter 1190, avg. loss 81.08, avg. ppl 43.43 cum. examples 6238, speed 1350.64 words/sec, time elapsed 316.86 sec\n",
            "epoch 3, iter 1200, avg. loss 80.98, avg. ppl 40.19 cum. examples 6398, speed 1437.36 words/sec, time elapsed 319.31 sec\n",
            "epoch 4, iter 1210, avg. loss 68.77, avg. ppl 26.00 cum. examples 6556, speed 1476.64 words/sec, time elapsed 321.56 sec\n",
            "epoch 4, iter 1220, avg. loss 71.75, avg. ppl 22.64 cum. examples 6716, speed 1340.29 words/sec, time elapsed 324.31 sec\n",
            "epoch 4, iter 1230, avg. loss 62.98, avg. ppl 19.70 cum. examples 6876, speed 1478.27 words/sec, time elapsed 326.60 sec\n",
            "epoch 4, iter 1240, avg. loss 70.60, avg. ppl 23.95 cum. examples 7036, speed 1148.97 words/sec, time elapsed 329.69 sec\n",
            "epoch 4, iter 1250, avg. loss 72.59, avg. ppl 22.89 cum. examples 7196, speed 1269.23 words/sec, time elapsed 332.62 sec\n",
            "epoch 4, iter 1260, avg. loss 67.51, avg. ppl 20.24 cum. examples 7356, speed 1323.49 words/sec, time elapsed 335.33 sec\n",
            "epoch 4, iter 1270, avg. loss 71.72, avg. ppl 23.96 cum. examples 7516, speed 1260.15 words/sec, time elapsed 338.20 sec\n",
            "epoch 4, iter 1280, avg. loss 63.42, avg. ppl 19.86 cum. examples 7676, speed 1229.36 words/sec, time elapsed 340.96 sec\n",
            "epoch 4, iter 1290, avg. loss 65.97, avg. ppl 21.56 cum. examples 7836, speed 1400.12 words/sec, time elapsed 343.41 sec\n",
            "epoch 4, iter 1300, avg. loss 78.78, avg. ppl 25.99 cum. examples 7996, speed 1049.51 words/sec, time elapsed 347.10 sec\n",
            "epoch 4, iter 1310, avg. loss 71.81, avg. ppl 23.51 cum. examples 8156, speed 1311.42 words/sec, time elapsed 349.87 sec\n",
            "epoch 4, iter 1320, avg. loss 58.98, avg. ppl 17.20 cum. examples 8316, speed 1329.49 words/sec, time elapsed 352.37 sec\n",
            "epoch 4, iter 1330, avg. loss 64.56, avg. ppl 22.48 cum. examples 8476, speed 1492.32 words/sec, time elapsed 354.59 sec\n",
            "epoch 4, iter 1340, avg. loss 68.24, avg. ppl 21.87 cum. examples 8636, speed 1413.93 words/sec, time elapsed 357.10 sec\n",
            "epoch 4, iter 1350, avg. loss 81.16, avg. ppl 27.53 cum. examples 8796, speed 1309.52 words/sec, time elapsed 360.09 sec\n",
            "epoch 4, iter 1360, avg. loss 69.12, avg. ppl 24.78 cum. examples 8956, speed 1439.48 words/sec, time elapsed 362.48 sec\n",
            "epoch 4, iter 1370, avg. loss 71.73, avg. ppl 25.45 cum. examples 9116, speed 1452.21 words/sec, time elapsed 364.92 sec\n",
            "epoch 4, iter 1380, avg. loss 75.39, avg. ppl 26.49 cum. examples 9276, speed 1366.09 words/sec, time elapsed 367.62 sec\n",
            "epoch 4, iter 1390, avg. loss 73.08, avg. ppl 24.66 cum. examples 9436, speed 1379.69 words/sec, time elapsed 370.26 sec\n",
            "epoch 4, iter 1400, avg. loss 77.71, avg. ppl 26.03 cum. examples 9596, speed 1377.53 words/sec, time elapsed 373.03 sec\n",
            "epoch 4, iter 1410, avg. loss 74.12, avg. ppl 26.09 cum. examples 9756, speed 1272.42 words/sec, time elapsed 375.89 sec\n",
            "epoch 4, iter 1420, avg. loss 73.69, avg. ppl 24.93 cum. examples 9916, speed 1192.33 words/sec, time elapsed 378.96 sec\n",
            "epoch 4, iter 1430, avg. loss 78.12, avg. ppl 27.36 cum. examples 10076, speed 1340.84 words/sec, time elapsed 381.78 sec\n",
            "epoch 4, iter 1440, avg. loss 71.06, avg. ppl 25.19 cum. examples 10236, speed 1517.61 words/sec, time elapsed 384.10 sec\n",
            "epoch 4, iter 1450, avg. loss 65.53, avg. ppl 24.12 cum. examples 10396, speed 1432.71 words/sec, time elapsed 386.40 sec\n",
            "epoch 4, iter 1460, avg. loss 72.36, avg. ppl 22.95 cum. examples 10556, speed 1287.22 words/sec, time elapsed 389.27 sec\n",
            "epoch 4, iter 1470, avg. loss 72.75, avg. ppl 26.54 cum. examples 10716, speed 1261.62 words/sec, time elapsed 392.09 sec\n",
            "epoch 4, iter 1480, avg. loss 67.85, avg. ppl 23.07 cum. examples 10876, speed 1534.28 words/sec, time elapsed 394.34 sec\n",
            "epoch 4, iter 1490, avg. loss 69.31, avg. ppl 24.32 cum. examples 11036, speed 1335.08 words/sec, time elapsed 396.94 sec\n",
            "epoch 4, iter 1500, avg. loss 65.81, avg. ppl 23.42 cum. examples 11196, speed 1744.40 words/sec, time elapsed 398.86 sec\n",
            "epoch 4, iter 1510, avg. loss 70.78, avg. ppl 25.05 cum. examples 11356, speed 1278.00 words/sec, time elapsed 401.61 sec\n",
            "epoch 4, iter 1520, avg. loss 71.82, avg. ppl 25.76 cum. examples 11516, speed 1257.37 words/sec, time elapsed 404.42 sec\n",
            "epoch 4, iter 1530, avg. loss 69.68, avg. ppl 25.04 cum. examples 11676, speed 1514.98 words/sec, time elapsed 406.71 sec\n",
            "epoch 4, iter 1540, avg. loss 65.14, avg. ppl 22.63 cum. examples 11836, speed 1532.65 words/sec, time elapsed 408.89 sec\n",
            "epoch 4, iter 1550, avg. loss 65.90, avg. ppl 22.93 cum. examples 11996, speed 1348.12 words/sec, time elapsed 411.38 sec\n",
            "epoch 4, iter 1560, avg. loss 68.81, avg. ppl 23.75 cum. examples 12156, speed 1415.56 words/sec, time elapsed 413.84 sec\n",
            "epoch 4, iter 1570, avg. loss 71.48, avg. ppl 26.80 cum. examples 12316, speed 1485.34 words/sec, time elapsed 416.18 sec\n",
            "epoch 4, iter 1580, avg. loss 71.11, avg. ppl 25.93 cum. examples 12476, speed 1339.61 words/sec, time elapsed 418.79 sec\n",
            "epoch 4, iter 1590, avg. loss 73.43, avg. ppl 27.62 cum. examples 12636, speed 1369.62 words/sec, time elapsed 421.37 sec\n",
            "epoch 4, iter 1600, avg. loss 68.73, avg. ppl 24.87 cum. examples 12796, speed 1449.07 words/sec, time elapsed 423.74 sec\n",
            "epoch 4, iter 1600, cum. loss 76.48, cum. ppl 31.77 cum. examples 12796\n",
            "begin validation ...\n",
            "validation: iter 1600, dev. ppl 64.534222\n",
            "hit patience 1\n",
            "epoch 5, iter 1610, avg. loss 54.57, avg. ppl 14.89 cum. examples 158, speed 397.90 words/sec, time elapsed 431.76 sec\n",
            "epoch 5, iter 1620, avg. loss 61.23, avg. ppl 14.04 cum. examples 318, speed 1285.80 words/sec, time elapsed 434.64 sec\n",
            "epoch 5, iter 1630, avg. loss 54.11, avg. ppl 12.62 cum. examples 478, speed 1377.09 words/sec, time elapsed 437.12 sec\n",
            "epoch 5, iter 1640, avg. loss 48.77, avg. ppl 10.53 cum. examples 638, speed 1343.85 words/sec, time elapsed 439.59 sec\n",
            "epoch 5, iter 1650, avg. loss 59.06, avg. ppl 12.95 cum. examples 798, speed 1310.84 words/sec, time elapsed 442.41 sec\n",
            "epoch 5, iter 1660, avg. loss 51.66, avg. ppl 11.06 cum. examples 958, speed 1602.33 words/sec, time elapsed 444.55 sec\n",
            "epoch 5, iter 1670, avg. loss 60.60, avg. ppl 13.94 cum. examples 1118, speed 1272.52 words/sec, time elapsed 447.44 sec\n",
            "epoch 5, iter 1680, avg. loss 57.17, avg. ppl 13.83 cum. examples 1278, speed 1383.71 words/sec, time elapsed 449.96 sec\n",
            "epoch 5, iter 1690, avg. loss 55.67, avg. ppl 12.43 cum. examples 1438, speed 1585.89 words/sec, time elapsed 452.19 sec\n",
            "epoch 5, iter 1700, avg. loss 60.94, avg. ppl 13.64 cum. examples 1598, speed 1197.30 words/sec, time elapsed 455.31 sec\n",
            "epoch 5, iter 1710, avg. loss 47.72, avg. ppl 10.30 cum. examples 1758, speed 1389.63 words/sec, time elapsed 457.66 sec\n",
            "epoch 5, iter 1720, avg. loss 55.59, avg. ppl 12.77 cum. examples 1918, speed 1128.60 words/sec, time elapsed 460.76 sec\n",
            "epoch 5, iter 1730, avg. loss 59.54, avg. ppl 13.52 cum. examples 2078, speed 1221.42 words/sec, time elapsed 463.75 sec\n",
            "epoch 5, iter 1740, avg. loss 57.35, avg. ppl 13.96 cum. examples 2238, speed 1452.76 words/sec, time elapsed 466.15 sec\n",
            "epoch 5, iter 1750, avg. loss 53.29, avg. ppl 12.54 cum. examples 2398, speed 1609.26 words/sec, time elapsed 468.24 sec\n",
            "epoch 5, iter 1760, avg. loss 59.24, avg. ppl 13.60 cum. examples 2558, speed 1308.00 words/sec, time elapsed 471.02 sec\n",
            "epoch 5, iter 1770, avg. loss 53.82, avg. ppl 11.84 cum. examples 2718, speed 1489.70 words/sec, time elapsed 473.36 sec\n",
            "epoch 5, iter 1780, avg. loss 57.32, avg. ppl 14.10 cum. examples 2878, speed 1352.31 words/sec, time elapsed 475.92 sec\n",
            "epoch 5, iter 1790, avg. loss 54.11, avg. ppl 12.46 cum. examples 3038, speed 1612.95 words/sec, time elapsed 478.05 sec\n",
            "epoch 5, iter 1800, avg. loss 54.79, avg. ppl 13.44 cum. examples 3198, speed 1191.82 words/sec, time elapsed 480.88 sec\n",
            "epoch 5, iter 1810, avg. loss 56.79, avg. ppl 13.87 cum. examples 3358, speed 1331.16 words/sec, time elapsed 483.47 sec\n",
            "epoch 5, iter 1820, avg. loss 51.12, avg. ppl 12.20 cum. examples 3518, speed 1460.34 words/sec, time elapsed 485.71 sec\n",
            "epoch 5, iter 1830, avg. loss 55.19, avg. ppl 13.68 cum. examples 3678, speed 1580.44 words/sec, time elapsed 487.85 sec\n",
            "epoch 5, iter 1840, avg. loss 55.75, avg. ppl 13.18 cum. examples 3838, speed 1639.78 words/sec, time elapsed 489.96 sec\n",
            "epoch 5, iter 1850, avg. loss 62.62, avg. ppl 15.50 cum. examples 3998, speed 1429.68 words/sec, time elapsed 492.52 sec\n",
            "epoch 5, iter 1860, avg. loss 61.05, avg. ppl 14.50 cum. examples 4158, speed 1180.42 words/sec, time elapsed 495.61 sec\n",
            "epoch 5, iter 1870, avg. loss 64.59, avg. ppl 16.21 cum. examples 4318, speed 1279.17 words/sec, time elapsed 498.51 sec\n",
            "epoch 5, iter 1880, avg. loss 67.63, avg. ppl 17.56 cum. examples 4478, speed 1428.45 words/sec, time elapsed 501.15 sec\n",
            "epoch 5, iter 1890, avg. loss 53.70, avg. ppl 13.23 cum. examples 4638, speed 1361.68 words/sec, time elapsed 503.60 sec\n",
            "epoch 5, iter 1900, avg. loss 53.40, avg. ppl 13.11 cum. examples 4798, speed 1333.08 words/sec, time elapsed 506.09 sec\n",
            "epoch 5, iter 1910, avg. loss 53.18, avg. ppl 13.38 cum. examples 4958, speed 1259.41 words/sec, time elapsed 508.69 sec\n",
            "epoch 5, iter 1920, avg. loss 57.93, avg. ppl 13.70 cum. examples 5118, speed 1569.75 words/sec, time elapsed 510.95 sec\n",
            "epoch 5, iter 1930, avg. loss 53.82, avg. ppl 13.70 cum. examples 5278, speed 1481.98 words/sec, time elapsed 513.17 sec\n",
            "epoch 5, iter 1940, avg. loss 63.14, avg. ppl 15.93 cum. examples 5438, speed 1195.02 words/sec, time elapsed 516.22 sec\n",
            "epoch 5, iter 1950, avg. loss 68.95, avg. ppl 17.88 cum. examples 5598, speed 1306.08 words/sec, time elapsed 519.15 sec\n",
            "epoch 5, iter 1960, avg. loss 61.20, avg. ppl 14.42 cum. examples 5758, speed 1453.19 words/sec, time elapsed 521.68 sec\n",
            "epoch 5, iter 1970, avg. loss 58.56, avg. ppl 14.62 cum. examples 5918, speed 1230.21 words/sec, time elapsed 524.52 sec\n",
            "epoch 5, iter 1980, avg. loss 66.74, avg. ppl 17.13 cum. examples 6078, speed 1308.43 words/sec, time elapsed 527.39 sec\n",
            "epoch 5, iter 1990, avg. loss 70.37, avg. ppl 17.60 cum. examples 6238, speed 1168.98 words/sec, time elapsed 530.75 sec\n",
            "epoch 5, iter 2000, avg. loss 70.59, avg. ppl 17.53 cum. examples 6398, speed 1491.00 words/sec, time elapsed 533.39 sec\n",
            "epoch 6, iter 2010, avg. loss 56.41, avg. ppl 11.19 cum. examples 6556, speed 1394.35 words/sec, time elapsed 536.04 sec\n",
            "epoch 6, iter 2020, avg. loss 36.68, avg. ppl 5.79 cum. examples 6716, speed 1508.54 words/sec, time elapsed 538.26 sec\n",
            "epoch 6, iter 2030, avg. loss 45.23, avg. ppl 7.77 cum. examples 6876, speed 1226.52 words/sec, time elapsed 541.13 sec\n",
            "epoch 6, iter 2040, avg. loss 45.87, avg. ppl 7.63 cum. examples 7036, speed 1348.32 words/sec, time elapsed 543.81 sec\n",
            "epoch 6, iter 2050, avg. loss 48.36, avg. ppl 8.28 cum. examples 7196, speed 1283.02 words/sec, time elapsed 546.67 sec\n",
            "epoch 6, iter 2060, avg. loss 45.67, avg. ppl 7.48 cum. examples 7356, speed 1431.40 words/sec, time elapsed 549.20 sec\n",
            "epoch 6, iter 2070, avg. loss 41.64, avg. ppl 7.15 cum. examples 7516, speed 1359.44 words/sec, time elapsed 551.69 sec\n",
            "epoch 6, iter 2080, avg. loss 46.12, avg. ppl 7.84 cum. examples 7676, speed 1509.13 words/sec, time elapsed 554.07 sec\n",
            "epoch 6, iter 2090, avg. loss 46.40, avg. ppl 7.62 cum. examples 7836, speed 1372.01 words/sec, time elapsed 556.73 sec\n",
            "epoch 6, iter 2100, avg. loss 43.07, avg. ppl 7.12 cum. examples 7996, speed 1598.90 words/sec, time elapsed 558.93 sec\n",
            "epoch 6, iter 2110, avg. loss 49.02, avg. ppl 8.33 cum. examples 8156, speed 1261.81 words/sec, time elapsed 561.86 sec\n",
            "epoch 6, iter 2120, avg. loss 43.86, avg. ppl 7.30 cum. examples 8316, speed 1328.88 words/sec, time elapsed 564.52 sec\n",
            "epoch 6, iter 2130, avg. loss 46.23, avg. ppl 7.61 cum. examples 8476, speed 1614.63 words/sec, time elapsed 566.78 sec\n",
            "epoch 6, iter 2140, avg. loss 47.33, avg. ppl 7.88 cum. examples 8636, speed 1507.33 words/sec, time elapsed 569.21 sec\n",
            "epoch 6, iter 2150, avg. loss 48.33, avg. ppl 8.19 cum. examples 8796, speed 1472.77 words/sec, time elapsed 571.71 sec\n",
            "epoch 6, iter 2160, avg. loss 46.49, avg. ppl 8.42 cum. examples 8956, speed 1172.96 words/sec, time elapsed 574.68 sec\n",
            "epoch 6, iter 2170, avg. loss 41.41, avg. ppl 7.25 cum. examples 9116, speed 1517.61 words/sec, time elapsed 576.88 sec\n",
            "epoch 6, iter 2180, avg. loss 46.78, avg. ppl 8.39 cum. examples 9276, speed 1344.48 words/sec, time elapsed 579.50 sec\n",
            "epoch 6, iter 2190, avg. loss 46.34, avg. ppl 8.42 cum. examples 9436, speed 1228.31 words/sec, time elapsed 582.33 sec\n",
            "epoch 6, iter 2200, avg. loss 42.80, avg. ppl 7.65 cum. examples 9596, speed 1317.40 words/sec, time elapsed 584.89 sec\n",
            "epoch 6, iter 2210, avg. loss 43.70, avg. ppl 7.74 cum. examples 9756, speed 1349.48 words/sec, time elapsed 587.42 sec\n",
            "epoch 6, iter 2220, avg. loss 42.44, avg. ppl 7.83 cum. examples 9916, speed 1367.82 words/sec, time elapsed 589.83 sec\n",
            "epoch 6, iter 2230, avg. loss 49.72, avg. ppl 9.69 cum. examples 10076, speed 1377.75 words/sec, time elapsed 592.37 sec\n",
            "epoch 6, iter 2240, avg. loss 36.46, avg. ppl 6.46 cum. examples 10236, speed 1512.05 words/sec, time elapsed 594.44 sec\n",
            "epoch 6, iter 2250, avg. loss 47.90, avg. ppl 9.03 cum. examples 10396, speed 1202.53 words/sec, time elapsed 597.34 sec\n",
            "epoch 6, iter 2260, avg. loss 46.55, avg. ppl 8.27 cum. examples 10556, speed 1450.64 words/sec, time elapsed 599.77 sec\n",
            "epoch 6, iter 2270, avg. loss 52.57, avg. ppl 9.34 cum. examples 10716, speed 1103.90 words/sec, time elapsed 603.18 sec\n",
            "epoch 6, iter 2280, avg. loss 53.88, avg. ppl 10.16 cum. examples 10876, speed 1319.77 words/sec, time elapsed 606.00 sec\n",
            "epoch 6, iter 2290, avg. loss 58.89, avg. ppl 11.04 cum. examples 11036, speed 1164.05 words/sec, time elapsed 609.37 sec\n",
            "epoch 6, iter 2300, avg. loss 48.70, avg. ppl 9.18 cum. examples 11196, speed 1382.97 words/sec, time elapsed 611.91 sec\n",
            "epoch 6, iter 2310, avg. loss 55.04, avg. ppl 9.84 cum. examples 11356, speed 1509.64 words/sec, time elapsed 614.46 sec\n",
            "epoch 6, iter 2320, avg. loss 48.53, avg. ppl 9.18 cum. examples 11516, speed 1328.14 words/sec, time elapsed 617.10 sec\n",
            "epoch 6, iter 2330, avg. loss 41.10, avg. ppl 7.53 cum. examples 11676, speed 1578.99 words/sec, time elapsed 619.16 sec\n",
            "epoch 6, iter 2340, avg. loss 56.38, avg. ppl 10.78 cum. examples 11836, speed 1411.73 words/sec, time elapsed 621.85 sec\n",
            "epoch 6, iter 2350, avg. loss 44.14, avg. ppl 8.54 cum. examples 11996, speed 1528.91 words/sec, time elapsed 624.00 sec\n",
            "epoch 6, iter 2360, avg. loss 47.34, avg. ppl 8.78 cum. examples 12156, speed 1483.53 words/sec, time elapsed 626.35 sec\n",
            "epoch 6, iter 2370, avg. loss 46.19, avg. ppl 8.75 cum. examples 12316, speed 1427.19 words/sec, time elapsed 628.74 sec\n",
            "epoch 6, iter 2380, avg. loss 41.26, avg. ppl 7.52 cum. examples 12476, speed 1415.82 words/sec, time elapsed 631.05 sec\n",
            "epoch 6, iter 2390, avg. loss 49.42, avg. ppl 9.03 cum. examples 12636, speed 1411.90 words/sec, time elapsed 633.60 sec\n",
            "epoch 6, iter 2400, avg. loss 51.67, avg. ppl 9.67 cum. examples 12796, speed 1215.58 words/sec, time elapsed 636.59 sec\n",
            "epoch 6, iter 2400, cum. loss 52.48, cum. ppl 10.76 cum. examples 12796\n",
            "begin validation ...\n",
            "validation: iter 2400, dev. ppl 95.888830\n",
            "hit patience 2\n",
            "epoch 7, iter 2410, avg. loss 43.11, avg. ppl 6.92 cum. examples 158, speed 415.17 words/sec, time elapsed 645.08 sec\n",
            "epoch 7, iter 2420, avg. loss 29.84, avg. ppl 4.34 cum. examples 318, speed 1375.02 words/sec, time elapsed 647.44 sec\n",
            "epoch 7, iter 2430, avg. loss 29.70, avg. ppl 4.18 cum. examples 478, speed 1574.51 words/sec, time elapsed 649.55 sec\n",
            "epoch 7, iter 2440, avg. loss 40.05, avg. ppl 5.21 cum. examples 638, speed 1509.16 words/sec, time elapsed 652.13 sec\n",
            "epoch 7, iter 2450, avg. loss 30.23, avg. ppl 4.32 cum. examples 798, speed 1597.41 words/sec, time elapsed 654.20 sec\n",
            "epoch 7, iter 2460, avg. loss 43.44, avg. ppl 5.88 cum. examples 958, speed 1352.62 words/sec, time elapsed 657.10 sec\n",
            "epoch 7, iter 2470, avg. loss 33.55, avg. ppl 4.70 cum. examples 1118, speed 1420.58 words/sec, time elapsed 659.54 sec\n",
            "epoch 7, iter 2480, avg. loss 32.48, avg. ppl 4.37 cum. examples 1278, speed 1432.62 words/sec, time elapsed 662.00 sec\n",
            "epoch 7, iter 2490, avg. loss 37.05, avg. ppl 5.13 cum. examples 1438, speed 1310.56 words/sec, time elapsed 664.76 sec\n",
            "epoch 7, iter 2500, avg. loss 33.62, avg. ppl 4.72 cum. examples 1598, speed 1347.84 words/sec, time elapsed 667.33 sec\n",
            "epoch 7, iter 2510, avg. loss 30.09, avg. ppl 4.19 cum. examples 1758, speed 1416.71 words/sec, time elapsed 669.70 sec\n",
            "epoch 7, iter 2520, avg. loss 36.85, avg. ppl 5.01 cum. examples 1918, speed 1557.59 words/sec, time elapsed 672.05 sec\n",
            "epoch 7, iter 2530, avg. loss 38.07, avg. ppl 5.43 cum. examples 2078, speed 1219.90 words/sec, time elapsed 675.01 sec\n",
            "epoch 7, iter 2540, avg. loss 40.84, avg. ppl 6.20 cum. examples 2238, speed 1189.52 words/sec, time elapsed 678.02 sec\n",
            "epoch 7, iter 2550, avg. loss 35.03, avg. ppl 5.18 cum. examples 2398, speed 1257.98 words/sec, time elapsed 680.73 sec\n",
            "epoch 7, iter 2560, avg. loss 37.00, avg. ppl 5.22 cum. examples 2558, speed 1372.75 words/sec, time elapsed 683.34 sec\n",
            "epoch 7, iter 2570, avg. loss 40.34, avg. ppl 5.78 cum. examples 2718, speed 1268.99 words/sec, time elapsed 686.24 sec\n",
            "epoch 7, iter 2580, avg. loss 38.80, avg. ppl 5.62 cum. examples 2878, speed 1338.56 words/sec, time elapsed 688.92 sec\n",
            "epoch 7, iter 2590, avg. loss 44.12, avg. ppl 6.16 cum. examples 3038, speed 1164.78 words/sec, time elapsed 692.26 sec\n",
            "epoch 7, iter 2600, avg. loss 36.48, avg. ppl 5.54 cum. examples 3198, speed 1245.36 words/sec, time elapsed 694.99 sec\n",
            "epoch 7, iter 2610, avg. loss 35.76, avg. ppl 5.12 cum. examples 3358, speed 1452.71 words/sec, time elapsed 697.40 sec\n",
            "epoch 7, iter 2620, avg. loss 34.61, avg. ppl 5.08 cum. examples 3518, speed 1329.81 words/sec, time elapsed 699.97 sec\n",
            "epoch 7, iter 2630, avg. loss 35.42, avg. ppl 5.04 cum. examples 3678, speed 1477.93 words/sec, time elapsed 702.34 sec\n",
            "epoch 7, iter 2640, avg. loss 30.05, avg. ppl 4.29 cum. examples 3838, speed 1472.40 words/sec, time elapsed 704.58 sec\n",
            "epoch 7, iter 2650, avg. loss 40.16, avg. ppl 5.75 cum. examples 3998, speed 1411.27 words/sec, time elapsed 707.18 sec\n",
            "epoch 7, iter 2660, avg. loss 34.01, avg. ppl 5.11 cum. examples 4158, speed 1400.45 words/sec, time elapsed 709.56 sec\n",
            "epoch 7, iter 2670, avg. loss 35.97, avg. ppl 5.40 cum. examples 4318, speed 1543.10 words/sec, time elapsed 711.77 sec\n",
            "epoch 7, iter 2680, avg. loss 37.39, avg. ppl 5.74 cum. examples 4478, speed 1393.57 words/sec, time elapsed 714.23 sec\n",
            "epoch 7, iter 2690, avg. loss 31.09, avg. ppl 4.81 cum. examples 4638, speed 1323.69 words/sec, time elapsed 716.62 sec\n",
            "epoch 7, iter 2700, avg. loss 38.09, avg. ppl 5.83 cum. examples 4798, speed 1203.09 words/sec, time elapsed 719.50 sec\n",
            "epoch 7, iter 2710, avg. loss 40.89, avg. ppl 5.85 cum. examples 4958, speed 1272.90 words/sec, time elapsed 722.41 sec\n",
            "epoch 7, iter 2720, avg. loss 42.84, avg. ppl 6.11 cum. examples 5118, speed 1466.40 words/sec, time elapsed 724.99 sec\n",
            "epoch 7, iter 2730, avg. loss 40.40, avg. ppl 5.97 cum. examples 5278, speed 1240.32 words/sec, time elapsed 727.91 sec\n",
            "epoch 7, iter 2740, avg. loss 38.65, avg. ppl 5.69 cum. examples 5438, speed 1270.59 words/sec, time elapsed 730.71 sec\n",
            "epoch 7, iter 2750, avg. loss 42.72, avg. ppl 6.72 cum. examples 5598, speed 1517.98 words/sec, time elapsed 733.07 sec\n",
            "epoch 7, iter 2760, avg. loss 37.46, avg. ppl 5.75 cum. examples 5758, speed 1220.73 words/sec, time elapsed 735.88 sec\n",
            "epoch 7, iter 2770, avg. loss 39.35, avg. ppl 5.89 cum. examples 5918, speed 1423.03 words/sec, time elapsed 738.37 sec\n",
            "epoch 7, iter 2780, avg. loss 37.21, avg. ppl 5.63 cum. examples 6078, speed 1548.14 words/sec, time elapsed 740.60 sec\n",
            "epoch 7, iter 2790, avg. loss 39.33, avg. ppl 5.95 cum. examples 6238, speed 1113.87 words/sec, time elapsed 743.76 sec\n",
            "epoch 7, iter 2800, avg. loss 41.93, avg. ppl 6.26 cum. examples 6398, speed 1412.49 words/sec, time elapsed 746.35 sec\n",
            "epoch 8, iter 2810, avg. loss 45.64, avg. ppl 6.33 cum. examples 6556, speed 1146.92 words/sec, time elapsed 749.76 sec\n",
            "epoch 8, iter 2820, avg. loss 28.72, avg. ppl 3.49 cum. examples 6716, speed 1403.85 words/sec, time elapsed 752.38 sec\n",
            "epoch 8, iter 2830, avg. loss 26.62, avg. ppl 3.30 cum. examples 6876, speed 1317.41 words/sec, time elapsed 755.08 sec\n",
            "epoch 8, iter 2840, avg. loss 29.18, avg. ppl 3.71 cum. examples 7036, speed 1329.20 words/sec, time elapsed 757.76 sec\n",
            "epoch 8, iter 2850, avg. loss 35.53, avg. ppl 4.31 cum. examples 7196, speed 1370.35 words/sec, time elapsed 760.60 sec\n",
            "epoch 8, iter 2860, avg. loss 32.33, avg. ppl 3.95 cum. examples 7356, speed 1257.18 words/sec, time elapsed 763.60 sec\n",
            "epoch 8, iter 2870, avg. loss 25.25, avg. ppl 3.15 cum. examples 7516, speed 1439.97 words/sec, time elapsed 766.04 sec\n",
            "epoch 8, iter 2880, avg. loss 27.23, avg. ppl 3.65 cum. examples 7676, speed 1212.94 words/sec, time elapsed 768.82 sec\n",
            "epoch 8, iter 2890, avg. loss 25.63, avg. ppl 3.36 cum. examples 7836, speed 1373.58 words/sec, time elapsed 771.28 sec\n",
            "epoch 8, iter 2900, avg. loss 28.34, avg. ppl 3.39 cum. examples 7996, speed 1396.70 words/sec, time elapsed 773.94 sec\n",
            "epoch 8, iter 2910, avg. loss 26.00, avg. ppl 3.34 cum. examples 8156, speed 1488.25 words/sec, time elapsed 776.26 sec\n",
            "epoch 8, iter 2920, avg. loss 25.48, avg. ppl 3.33 cum. examples 8316, speed 1541.85 words/sec, time elapsed 778.46 sec\n",
            "epoch 8, iter 2930, avg. loss 31.39, avg. ppl 4.06 cum. examples 8476, speed 1214.52 words/sec, time elapsed 781.41 sec\n",
            "epoch 8, iter 2940, avg. loss 22.86, avg. ppl 3.05 cum. examples 8636, speed 1481.59 words/sec, time elapsed 783.63 sec\n",
            "epoch 8, iter 2950, avg. loss 32.08, avg. ppl 4.06 cum. examples 8796, speed 1379.23 words/sec, time elapsed 786.28 sec\n",
            "epoch 8, iter 2960, avg. loss 32.72, avg. ppl 3.96 cum. examples 8956, speed 1299.17 words/sec, time elapsed 789.21 sec\n",
            "epoch 8, iter 2970, avg. loss 25.29, avg. ppl 3.36 cum. examples 9116, speed 1271.56 words/sec, time elapsed 791.83 sec\n",
            "epoch 8, iter 2980, avg. loss 26.39, avg. ppl 3.32 cum. examples 9276, speed 1438.41 words/sec, time elapsed 794.28 sec\n",
            "epoch 8, iter 2990, avg. loss 33.47, avg. ppl 4.38 cum. examples 9436, speed 1271.54 words/sec, time elapsed 797.14 sec\n",
            "epoch 8, iter 3000, avg. loss 31.60, avg. ppl 3.89 cum. examples 9596, speed 1397.46 words/sec, time elapsed 799.80 sec\n",
            "epoch 8, iter 3010, avg. loss 24.61, avg. ppl 3.21 cum. examples 9756, speed 1502.49 words/sec, time elapsed 802.05 sec\n",
            "epoch 8, iter 3020, avg. loss 28.62, avg. ppl 3.61 cum. examples 9916, speed 1402.16 words/sec, time elapsed 804.59 sec\n",
            "epoch 8, iter 3030, avg. loss 30.02, avg. ppl 4.05 cum. examples 10076, speed 1250.30 words/sec, time elapsed 807.34 sec\n",
            "epoch 8, iter 3040, avg. loss 19.61, avg. ppl 2.69 cum. examples 10236, speed 1607.66 words/sec, time elapsed 809.31 sec\n",
            "epoch 8, iter 3050, avg. loss 29.83, avg. ppl 3.75 cum. examples 10396, speed 1401.66 words/sec, time elapsed 811.89 sec\n",
            "epoch 8, iter 3060, avg. loss 37.60, avg. ppl 4.59 cum. examples 10556, speed 1275.07 words/sec, time elapsed 814.98 sec\n",
            "epoch 8, iter 3070, avg. loss 28.03, avg. ppl 3.62 cum. examples 10716, speed 1428.22 words/sec, time elapsed 817.42 sec\n",
            "epoch 8, iter 3080, avg. loss 34.35, avg. ppl 4.29 cum. examples 10876, speed 1290.31 words/sec, time elapsed 820.35 sec\n",
            "epoch 8, iter 3090, avg. loss 22.88, avg. ppl 3.02 cum. examples 11036, speed 1457.49 words/sec, time elapsed 822.62 sec\n",
            "epoch 8, iter 3100, avg. loss 29.70, avg. ppl 4.04 cum. examples 11196, speed 1193.39 words/sec, time elapsed 825.48 sec\n",
            "epoch 8, iter 3110, avg. loss 28.62, avg. ppl 3.90 cum. examples 11356, speed 1380.25 words/sec, time elapsed 827.92 sec\n",
            "epoch 8, iter 3120, avg. loss 32.66, avg. ppl 4.28 cum. examples 11516, speed 1247.18 words/sec, time elapsed 830.80 sec\n",
            "epoch 8, iter 3130, avg. loss 32.45, avg. ppl 4.10 cum. examples 11676, speed 1424.59 words/sec, time elapsed 833.38 sec\n",
            "epoch 8, iter 3140, avg. loss 25.01, avg. ppl 3.34 cum. examples 11836, speed 1507.04 words/sec, time elapsed 835.58 sec\n",
            "epoch 8, iter 3150, avg. loss 28.31, avg. ppl 3.76 cum. examples 11996, speed 1495.38 words/sec, time elapsed 837.87 sec\n",
            "epoch 8, iter 3160, avg. loss 31.88, avg. ppl 4.17 cum. examples 12156, speed 1325.28 words/sec, time elapsed 840.56 sec\n",
            "epoch 8, iter 3170, avg. loss 27.06, avg. ppl 3.61 cum. examples 12316, speed 1409.08 words/sec, time elapsed 842.96 sec\n",
            "epoch 8, iter 3180, avg. loss 33.02, avg. ppl 4.29 cum. examples 12476, speed 1525.15 words/sec, time elapsed 845.33 sec\n",
            "epoch 8, iter 3190, avg. loss 30.97, avg. ppl 4.02 cum. examples 12636, speed 1432.92 words/sec, time elapsed 847.82 sec\n",
            "epoch 8, iter 3200, avg. loss 35.99, avg. ppl 4.84 cum. examples 12796, speed 1224.14 words/sec, time elapsed 850.80 sec\n",
            "epoch 8, iter 3200, cum. loss 33.33, cum. ppl 4.51 cum. examples 12796\n",
            "begin validation ...\n",
            "validation: iter 3200, dev. ppl 166.002495\n",
            "hit patience 3\n",
            "epoch 9, iter 3210, avg. loss 28.09, avg. ppl 3.62 cum. examples 158, speed 418.19 words/sec, time elapsed 859.05 sec\n",
            "epoch 9, iter 3220, avg. loss 22.25, avg. ppl 2.73 cum. examples 318, speed 1269.40 words/sec, time elapsed 861.84 sec\n",
            "epoch 9, iter 3230, avg. loss 23.81, avg. ppl 2.78 cum. examples 478, speed 1374.42 words/sec, time elapsed 864.55 sec\n",
            "epoch 9, iter 3240, avg. loss 18.82, avg. ppl 2.47 cum. examples 638, speed 1553.60 words/sec, time elapsed 866.69 sec\n",
            "epoch 9, iter 3250, avg. loss 19.62, avg. ppl 2.56 cum. examples 798, speed 1482.24 words/sec, time elapsed 868.94 sec\n",
            "epoch 9, iter 3260, avg. loss 24.21, avg. ppl 2.74 cum. examples 958, speed 1497.96 words/sec, time elapsed 871.51 sec\n",
            "epoch 9, iter 3270, avg. loss 16.64, avg. ppl 2.26 cum. examples 1118, speed 1387.94 words/sec, time elapsed 873.85 sec\n",
            "epoch 9, iter 3280, avg. loss 19.62, avg. ppl 2.48 cum. examples 1278, speed 1366.94 words/sec, time elapsed 876.39 sec\n",
            "epoch 9, iter 3290, avg. loss 21.43, avg. ppl 2.74 cum. examples 1438, speed 1205.99 words/sec, time elapsed 879.20 sec\n",
            "epoch 9, iter 3300, avg. loss 20.07, avg. ppl 2.54 cum. examples 1598, speed 1398.51 words/sec, time elapsed 881.66 sec\n",
            "epoch 9, iter 3310, avg. loss 18.82, avg. ppl 2.40 cum. examples 1758, speed 1583.34 words/sec, time elapsed 883.84 sec\n",
            "epoch 9, iter 3320, avg. loss 21.31, avg. ppl 2.64 cum. examples 1918, speed 1306.99 words/sec, time elapsed 886.53 sec\n",
            "epoch 9, iter 3330, avg. loss 16.39, avg. ppl 2.24 cum. examples 2078, speed 1506.47 words/sec, time elapsed 888.68 sec\n",
            "epoch 9, iter 3340, avg. loss 23.56, avg. ppl 2.72 cum. examples 2238, speed 1428.09 words/sec, time elapsed 891.32 sec\n",
            "epoch 9, iter 3350, avg. loss 27.19, avg. ppl 3.14 cum. examples 2398, speed 1272.52 words/sec, time elapsed 894.31 sec\n",
            "epoch 9, iter 3360, avg. loss 19.99, avg. ppl 2.59 cum. examples 2558, speed 1317.54 words/sec, time elapsed 896.86 sec\n",
            "epoch 9, iter 3370, avg. loss 25.04, avg. ppl 3.00 cum. examples 2718, speed 1241.66 words/sec, time elapsed 899.80 sec\n",
            "epoch 9, iter 3380, avg. loss 21.82, avg. ppl 2.75 cum. examples 2878, speed 1210.83 words/sec, time elapsed 902.65 sec\n",
            "epoch 9, iter 3390, avg. loss 24.95, avg. ppl 2.99 cum. examples 3038, speed 1299.15 words/sec, time elapsed 905.45 sec\n",
            "epoch 9, iter 3400, avg. loss 21.36, avg. ppl 2.68 cum. examples 3198, speed 1352.48 words/sec, time elapsed 908.02 sec\n",
            "epoch 9, iter 3410, avg. loss 22.11, avg. ppl 2.67 cum. examples 3358, speed 1342.30 words/sec, time elapsed 910.70 sec\n",
            "epoch 9, iter 3420, avg. loss 23.40, avg. ppl 2.80 cum. examples 3518, speed 1475.99 words/sec, time elapsed 913.16 sec\n",
            "epoch 9, iter 3430, avg. loss 26.28, avg. ppl 3.27 cum. examples 3678, speed 1131.94 words/sec, time elapsed 916.30 sec\n",
            "epoch 9, iter 3440, avg. loss 18.32, avg. ppl 2.46 cum. examples 3838, speed 1508.62 words/sec, time elapsed 918.46 sec\n",
            "epoch 9, iter 3450, avg. loss 26.09, avg. ppl 3.05 cum. examples 3998, speed 1199.63 words/sec, time elapsed 921.58 sec\n",
            "epoch 9, iter 3460, avg. loss 22.71, avg. ppl 2.82 cum. examples 4158, speed 1377.55 words/sec, time elapsed 924.12 sec\n",
            "epoch 9, iter 3470, avg. loss 29.38, avg. ppl 3.43 cum. examples 4318, speed 1279.79 words/sec, time elapsed 927.10 sec\n",
            "epoch 9, iter 3480, avg. loss 27.67, avg. ppl 3.16 cum. examples 4478, speed 1266.42 words/sec, time elapsed 930.14 sec\n",
            "epoch 9, iter 3490, avg. loss 28.27, avg. ppl 3.24 cum. examples 4638, speed 1338.32 words/sec, time elapsed 933.02 sec\n",
            "epoch 9, iter 3500, avg. loss 23.43, avg. ppl 2.91 cum. examples 4798, speed 1374.42 words/sec, time elapsed 935.57 sec\n",
            "epoch 9, iter 3510, avg. loss 20.12, avg. ppl 2.58 cum. examples 4958, speed 1314.60 words/sec, time elapsed 938.15 sec\n",
            "epoch 9, iter 3520, avg. loss 26.13, avg. ppl 3.11 cum. examples 5118, speed 1332.97 words/sec, time elapsed 940.91 sec\n",
            "epoch 9, iter 3530, avg. loss 25.30, avg. ppl 3.01 cum. examples 5278, speed 1594.15 words/sec, time elapsed 943.21 sec\n",
            "epoch 9, iter 3540, avg. loss 25.30, avg. ppl 3.23 cum. examples 5438, speed 1138.58 words/sec, time elapsed 946.25 sec\n",
            "epoch 9, iter 3550, avg. loss 22.06, avg. ppl 2.70 cum. examples 5598, speed 1420.92 words/sec, time elapsed 948.75 sec\n",
            "epoch 9, iter 3560, avg. loss 26.79, avg. ppl 3.24 cum. examples 5758, speed 1257.53 words/sec, time elapsed 951.65 sec\n",
            "epoch 9, iter 3570, avg. loss 24.46, avg. ppl 2.95 cum. examples 5918, speed 1444.68 words/sec, time elapsed 954.16 sec\n",
            "epoch 9, iter 3580, avg. loss 20.55, avg. ppl 2.64 cum. examples 6078, speed 1673.87 words/sec, time elapsed 956.18 sec\n",
            "epoch 9, iter 3590, avg. loss 20.32, avg. ppl 2.73 cum. examples 6238, speed 1466.65 words/sec, time elapsed 958.39 sec\n",
            "epoch 9, iter 3600, avg. loss 25.65, avg. ppl 3.25 cum. examples 6398, speed 1331.84 words/sec, time elapsed 961.00 sec\n",
            "epoch 10, iter 3610, avg. loss 17.88, avg. ppl 2.47 cum. examples 6556, speed 1384.10 words/sec, time elapsed 963.26 sec\n",
            "epoch 10, iter 3620, avg. loss 16.66, avg. ppl 2.12 cum. examples 6716, speed 1410.81 words/sec, time elapsed 965.77 sec\n",
            "epoch 10, iter 3630, avg. loss 14.46, avg. ppl 2.00 cum. examples 6876, speed 1395.95 words/sec, time elapsed 968.16 sec\n",
            "epoch 10, iter 3640, avg. loss 14.47, avg. ppl 1.97 cum. examples 7036, speed 1509.84 words/sec, time elapsed 970.42 sec\n",
            "epoch 10, iter 3650, avg. loss 14.90, avg. ppl 1.97 cum. examples 7196, speed 1536.42 words/sec, time elapsed 972.70 sec\n",
            "epoch 10, iter 3660, avg. loss 19.44, avg. ppl 2.44 cum. examples 7356, speed 1132.75 words/sec, time elapsed 975.78 sec\n",
            "epoch 10, iter 3670, avg. loss 17.74, avg. ppl 2.18 cum. examples 7516, speed 1449.33 words/sec, time elapsed 978.29 sec\n",
            "epoch 10, iter 3680, avg. loss 11.57, avg. ppl 1.77 cum. examples 7676, speed 1443.45 words/sec, time elapsed 980.54 sec\n",
            "epoch 10, iter 3690, avg. loss 13.25, avg. ppl 1.88 cum. examples 7836, speed 1327.77 words/sec, time elapsed 983.07 sec\n",
            "epoch 10, iter 3700, avg. loss 18.95, avg. ppl 2.31 cum. examples 7996, speed 1229.41 words/sec, time elapsed 986.01 sec\n",
            "epoch 10, iter 3710, avg. loss 17.64, avg. ppl 2.21 cum. examples 8156, speed 1243.20 words/sec, time elapsed 988.86 sec\n",
            "epoch 10, iter 3720, avg. loss 15.78, avg. ppl 2.19 cum. examples 8316, speed 1142.29 words/sec, time elapsed 991.69 sec\n",
            "epoch 10, iter 3730, avg. loss 14.05, avg. ppl 2.03 cum. examples 8476, speed 1498.95 words/sec, time elapsed 993.81 sec\n",
            "epoch 10, iter 3740, avg. loss 18.36, avg. ppl 2.23 cum. examples 8636, speed 1402.56 words/sec, time elapsed 996.43 sec\n",
            "epoch 10, iter 3750, avg. loss 15.17, avg. ppl 2.05 cum. examples 8796, speed 1376.34 words/sec, time elapsed 998.88 sec\n",
            "epoch 10, iter 3760, avg. loss 16.47, avg. ppl 2.14 cum. examples 8956, speed 1322.29 words/sec, time elapsed 1001.50 sec\n",
            "epoch 10, iter 3770, avg. loss 19.24, avg. ppl 2.37 cum. examples 9116, speed 1280.31 words/sec, time elapsed 1004.29 sec\n",
            "epoch 10, iter 3780, avg. loss 17.95, avg. ppl 2.24 cum. examples 9276, speed 1405.80 words/sec, time elapsed 1006.82 sec\n",
            "epoch 10, iter 3790, avg. loss 15.42, avg. ppl 2.07 cum. examples 9436, speed 1502.85 words/sec, time elapsed 1009.08 sec\n",
            "epoch 10, iter 3800, avg. loss 20.81, avg. ppl 2.45 cum. examples 9596, speed 1241.25 words/sec, time elapsed 1012.06 sec\n",
            "epoch 10, iter 3810, avg. loss 13.35, avg. ppl 1.95 cum. examples 9756, speed 1493.32 words/sec, time elapsed 1014.21 sec\n",
            "epoch 10, iter 3820, avg. loss 18.99, avg. ppl 2.24 cum. examples 9916, speed 1407.67 words/sec, time elapsed 1016.89 sec\n",
            "epoch 10, iter 3830, avg. loss 15.62, avg. ppl 2.00 cum. examples 10076, speed 1531.81 words/sec, time elapsed 1019.24 sec\n",
            "epoch 10, iter 3840, avg. loss 16.40, avg. ppl 2.07 cum. examples 10236, speed 1495.77 words/sec, time elapsed 1021.65 sec\n",
            "epoch 10, iter 3850, avg. loss 16.70, avg. ppl 2.15 cum. examples 10396, speed 1270.55 words/sec, time elapsed 1024.40 sec\n",
            "epoch 10, iter 3860, avg. loss 22.94, avg. ppl 2.62 cum. examples 10556, speed 1199.86 words/sec, time elapsed 1027.58 sec\n",
            "epoch 10, iter 3870, avg. loss 20.91, avg. ppl 2.59 cum. examples 10716, speed 1212.22 words/sec, time elapsed 1030.48 sec\n",
            "epoch 10, iter 3880, avg. loss 19.07, avg. ppl 2.36 cum. examples 10876, speed 1424.87 words/sec, time elapsed 1032.98 sec\n",
            "epoch 10, iter 3890, avg. loss 19.20, avg. ppl 2.33 cum. examples 11036, speed 1292.07 words/sec, time elapsed 1035.79 sec\n",
            "epoch 10, iter 3900, avg. loss 18.74, avg. ppl 2.26 cum. examples 11196, speed 1356.72 words/sec, time elapsed 1038.49 sec\n",
            "epoch 10, iter 3910, avg. loss 22.97, avg. ppl 2.56 cum. examples 11356, speed 1330.66 words/sec, time elapsed 1041.44 sec\n",
            "epoch 10, iter 3920, avg. loss 16.36, avg. ppl 2.14 cum. examples 11516, speed 1563.55 words/sec, time elapsed 1043.64 sec\n",
            "epoch 10, iter 3930, avg. loss 21.10, avg. ppl 2.48 cum. examples 11676, speed 1411.91 words/sec, time elapsed 1046.27 sec\n",
            "epoch 10, iter 3940, avg. loss 17.15, avg. ppl 2.18 cum. examples 11836, speed 1434.21 words/sec, time elapsed 1048.72 sec\n",
            "epoch 10, iter 3950, avg. loss 25.14, avg. ppl 2.82 cum. examples 11996, speed 1237.73 words/sec, time elapsed 1051.85 sec\n",
            "epoch 10, iter 3960, avg. loss 19.59, avg. ppl 2.45 cum. examples 12156, speed 1433.29 words/sec, time elapsed 1054.30 sec\n",
            "epoch 10, iter 3970, avg. loss 20.61, avg. ppl 2.44 cum. examples 12316, speed 1286.04 words/sec, time elapsed 1057.17 sec\n",
            "epoch 10, iter 3980, avg. loss 21.07, avg. ppl 2.49 cum. examples 12476, speed 1482.47 words/sec, time elapsed 1059.66 sec\n",
            "epoch 10, iter 3990, avg. loss 17.33, avg. ppl 2.14 cum. examples 12636, speed 1532.81 words/sec, time elapsed 1062.04 sec\n",
            "epoch 10, iter 4000, avg. loss 17.24, avg. ppl 2.23 cum. examples 12796, speed 1340.58 words/sec, time elapsed 1064.61 sec\n",
            "epoch 10, iter 4000, cum. loss 20.37, cum. ppl 2.51 cum. examples 12796\n",
            "begin validation ...\n",
            "validation: iter 4000, dev. ppl 273.135649\n",
            "hit patience 4\n",
            "epoch 10, iter 4010, avg. loss 16.78, avg. ppl 2.21 cum. examples 158, speed 401.06 words/sec, time elapsed 1072.95 sec\n",
            "epoch 11, iter 4020, avg. loss 10.51, avg. ppl 1.61 cum. examples 318, speed 1398.56 words/sec, time elapsed 1075.48 sec\n",
            "epoch 11, iter 4030, avg. loss 12.90, avg. ppl 1.79 cum. examples 478, speed 1180.07 words/sec, time elapsed 1078.48 sec\n",
            "epoch 11, iter 4040, avg. loss 11.96, avg. ppl 1.74 cum. examples 638, speed 1299.61 words/sec, time elapsed 1081.12 sec\n",
            "epoch 11, iter 4050, avg. loss 7.67, avg. ppl 1.48 cum. examples 798, speed 1495.30 words/sec, time elapsed 1083.21 sec\n",
            "epoch 11, iter 4060, avg. loss 14.70, avg. ppl 1.93 cum. examples 958, speed 1304.85 words/sec, time elapsed 1085.96 sec\n",
            "epoch 11, iter 4070, avg. loss 10.13, avg. ppl 1.62 cum. examples 1118, speed 1441.32 words/sec, time elapsed 1088.30 sec\n",
            "epoch 11, iter 4080, avg. loss 9.82, avg. ppl 1.57 cum. examples 1278, speed 1794.67 words/sec, time elapsed 1090.24 sec\n",
            "epoch 11, iter 4090, avg. loss 10.51, avg. ppl 1.67 cum. examples 1438, speed 1461.56 words/sec, time elapsed 1092.48 sec\n",
            "epoch 11, iter 4100, avg. loss 14.90, avg. ppl 1.93 cum. examples 1598, speed 1327.50 words/sec, time elapsed 1095.21 sec\n",
            "epoch 11, iter 4110, avg. loss 11.89, avg. ppl 1.72 cum. examples 1758, speed 1389.44 words/sec, time elapsed 1097.74 sec\n",
            "epoch 11, iter 4120, avg. loss 10.48, avg. ppl 1.67 cum. examples 1918, speed 1321.86 words/sec, time elapsed 1100.23 sec\n",
            "epoch 11, iter 4130, avg. loss 9.97, avg. ppl 1.59 cum. examples 2078, speed 1497.22 words/sec, time elapsed 1102.53 sec\n",
            "epoch 11, iter 4140, avg. loss 15.83, avg. ppl 1.97 cum. examples 2238, speed 1210.55 words/sec, time elapsed 1105.62 sec\n",
            "epoch 11, iter 4150, avg. loss 17.66, avg. ppl 2.06 cum. examples 2398, speed 1281.32 words/sec, time elapsed 1108.68 sec\n",
            "epoch 11, iter 4160, avg. loss 15.19, avg. ppl 1.90 cum. examples 2558, speed 1306.44 words/sec, time elapsed 1111.58 sec\n",
            "epoch 11, iter 4170, avg. loss 17.74, avg. ppl 2.16 cum. examples 2718, speed 1045.37 words/sec, time elapsed 1115.11 sec\n",
            "epoch 11, iter 4180, avg. loss 15.35, avg. ppl 1.96 cum. examples 2878, speed 1338.93 words/sec, time elapsed 1117.83 sec\n",
            "epoch 11, iter 4190, avg. loss 12.99, avg. ppl 1.86 cum. examples 3038, speed 1301.78 words/sec, time elapsed 1120.39 sec\n",
            "epoch 11, iter 4200, avg. loss 9.82, avg. ppl 1.62 cum. examples 3198, speed 1435.17 words/sec, time elapsed 1122.66 sec\n",
            "epoch 11, iter 4210, avg. loss 14.67, avg. ppl 1.89 cum. examples 3358, speed 1541.89 words/sec, time elapsed 1125.06 sec\n",
            "epoch 11, iter 4220, avg. loss 15.18, avg. ppl 2.01 cum. examples 3518, speed 1168.96 words/sec, time elapsed 1128.02 sec\n",
            "epoch 11, iter 4230, avg. loss 18.44, avg. ppl 2.16 cum. examples 3678, speed 1224.53 words/sec, time elapsed 1131.14 sec\n",
            "epoch 11, iter 4240, avg. loss 14.18, avg. ppl 1.85 cum. examples 3838, speed 1501.38 words/sec, time elapsed 1133.61 sec\n",
            "epoch 11, iter 4250, avg. loss 17.86, avg. ppl 2.08 cum. examples 3998, speed 1376.43 words/sec, time elapsed 1136.44 sec\n",
            "epoch 11, iter 4260, avg. loss 9.17, avg. ppl 1.59 cum. examples 4158, speed 1522.50 words/sec, time elapsed 1138.51 sec\n",
            "epoch 11, iter 4270, avg. loss 15.82, avg. ppl 2.03 cum. examples 4318, speed 1267.33 words/sec, time elapsed 1141.32 sec\n",
            "epoch 11, iter 4280, avg. loss 10.55, avg. ppl 1.68 cum. examples 4478, speed 1486.86 words/sec, time elapsed 1143.49 sec\n",
            "epoch 11, iter 4290, avg. loss 13.95, avg. ppl 1.88 cum. examples 4638, speed 1430.14 words/sec, time elapsed 1145.98 sec\n",
            "epoch 11, iter 4300, avg. loss 13.10, avg. ppl 1.80 cum. examples 4798, speed 1460.15 words/sec, time elapsed 1148.43 sec\n",
            "epoch 11, iter 4310, avg. loss 13.32, avg. ppl 1.83 cum. examples 4958, speed 1371.33 words/sec, time elapsed 1151.00 sec\n",
            "epoch 11, iter 4320, avg. loss 17.40, avg. ppl 2.09 cum. examples 5118, speed 1346.80 words/sec, time elapsed 1153.80 sec\n",
            "epoch 11, iter 4330, avg. loss 15.15, avg. ppl 2.01 cum. examples 5278, speed 1310.39 words/sec, time elapsed 1156.45 sec\n",
            "epoch 11, iter 4340, avg. loss 16.61, avg. ppl 2.11 cum. examples 5438, speed 1292.51 words/sec, time elapsed 1159.21 sec\n",
            "epoch 11, iter 4350, avg. loss 11.36, avg. ppl 1.72 cum. examples 5598, speed 1653.62 words/sec, time elapsed 1161.24 sec\n",
            "epoch 11, iter 4360, avg. loss 21.18, avg. ppl 2.48 cum. examples 5758, speed 1259.26 words/sec, time elapsed 1164.20 sec\n",
            "epoch 11, iter 4370, avg. loss 17.87, avg. ppl 2.27 cum. examples 5918, speed 1407.38 words/sec, time elapsed 1166.67 sec\n",
            "epoch 11, iter 4380, avg. loss 15.39, avg. ppl 2.01 cum. examples 6078, speed 1227.21 words/sec, time elapsed 1169.56 sec\n",
            "epoch 11, iter 4390, avg. loss 13.98, avg. ppl 1.90 cum. examples 6238, speed 1473.63 words/sec, time elapsed 1171.92 sec\n",
            "epoch 11, iter 4400, avg. loss 17.07, avg. ppl 2.07 cum. examples 6398, speed 1367.63 words/sec, time elapsed 1174.67 sec\n",
            "epoch 11, iter 4410, avg. loss 16.15, avg. ppl 2.10 cum. examples 6558, speed 1211.98 words/sec, time elapsed 1177.54 sec\n",
            "epoch 12, iter 4420, avg. loss 15.12, avg. ppl 1.88 cum. examples 6716, speed 1175.68 words/sec, time elapsed 1180.77 sec\n",
            "epoch 12, iter 4430, avg. loss 8.99, avg. ppl 1.53 cum. examples 6876, speed 1479.96 words/sec, time elapsed 1183.04 sec\n",
            "epoch 12, iter 4440, avg. loss 13.27, avg. ppl 1.76 cum. examples 7036, speed 1392.25 words/sec, time elapsed 1185.73 sec\n",
            "epoch 12, iter 4450, avg. loss 10.85, avg. ppl 1.60 cum. examples 7196, speed 1391.48 words/sec, time elapsed 1188.38 sec\n",
            "epoch 12, iter 4460, avg. loss 14.44, avg. ppl 1.79 cum. examples 7356, speed 1352.20 words/sec, time elapsed 1191.32 sec\n",
            "epoch 12, iter 4470, avg. loss 11.12, avg. ppl 1.64 cum. examples 7516, speed 1325.86 words/sec, time elapsed 1194.03 sec\n",
            "epoch 12, iter 4480, avg. loss 10.86, avg. ppl 1.63 cum. examples 7676, speed 1370.35 words/sec, time elapsed 1196.61 sec\n",
            "epoch 12, iter 4490, avg. loss 10.75, avg. ppl 1.60 cum. examples 7836, speed 1388.72 words/sec, time elapsed 1199.25 sec\n",
            "epoch 12, iter 4500, avg. loss 9.02, avg. ppl 1.56 cum. examples 7996, speed 1405.13 words/sec, time elapsed 1201.57 sec\n",
            "epoch 12, iter 4510, avg. loss 14.43, avg. ppl 1.87 cum. examples 8156, speed 1105.03 words/sec, time elapsed 1204.91 sec\n",
            "epoch 12, iter 4520, avg. loss 12.27, avg. ppl 1.73 cum. examples 8316, speed 1306.77 words/sec, time elapsed 1207.65 sec\n",
            "epoch 12, iter 4530, avg. loss 10.88, avg. ppl 1.62 cum. examples 8476, speed 1331.66 words/sec, time elapsed 1210.37 sec\n",
            "epoch 12, iter 4540, avg. loss 12.11, avg. ppl 1.72 cum. examples 8636, speed 1239.12 words/sec, time elapsed 1213.27 sec\n",
            "epoch 12, iter 4550, avg. loss 9.37, avg. ppl 1.55 cum. examples 8796, speed 1449.47 words/sec, time elapsed 1215.62 sec\n",
            "epoch 12, iter 4560, avg. loss 12.04, avg. ppl 1.67 cum. examples 8956, speed 1357.35 words/sec, time elapsed 1218.39 sec\n",
            "epoch 12, iter 4570, avg. loss 12.10, avg. ppl 1.67 cum. examples 9116, speed 1391.82 words/sec, time elapsed 1221.11 sec\n",
            "epoch 12, iter 4580, avg. loss 12.51, avg. ppl 1.81 cum. examples 9276, speed 1126.18 words/sec, time elapsed 1224.12 sec\n",
            "epoch 12, iter 4590, avg. loss 6.75, avg. ppl 1.42 cum. examples 9436, speed 1179.63 words/sec, time elapsed 1226.70 sec\n",
            "epoch 12, iter 4600, avg. loss 8.33, avg. ppl 1.49 cum. examples 9596, speed 1486.15 words/sec, time elapsed 1228.93 sec\n",
            "epoch 12, iter 4610, avg. loss 6.90, avg. ppl 1.42 cum. examples 9756, speed 1285.21 words/sec, time elapsed 1231.40 sec\n",
            "epoch 12, iter 4620, avg. loss 8.88, avg. ppl 1.52 cum. examples 9916, speed 1411.77 words/sec, time elapsed 1233.79 sec\n",
            "epoch 12, iter 4630, avg. loss 8.99, avg. ppl 1.53 cum. examples 10076, speed 1586.26 words/sec, time elapsed 1235.90 sec\n",
            "epoch 12, iter 4640, avg. loss 13.16, avg. ppl 1.75 cum. examples 10236, speed 1307.23 words/sec, time elapsed 1238.79 sec\n",
            "epoch 12, iter 4650, avg. loss 7.10, avg. ppl 1.42 cum. examples 10396, speed 1612.19 words/sec, time elapsed 1240.81 sec\n",
            "epoch 12, iter 4660, avg. loss 9.60, avg. ppl 1.59 cum. examples 10556, speed 1372.06 words/sec, time elapsed 1243.23 sec\n",
            "epoch 12, iter 4670, avg. loss 11.49, avg. ppl 1.68 cum. examples 10716, speed 1392.50 words/sec, time elapsed 1245.77 sec\n",
            "epoch 12, iter 4680, avg. loss 10.41, avg. ppl 1.60 cum. examples 10876, speed 1360.19 words/sec, time elapsed 1248.37 sec\n",
            "epoch 12, iter 4690, avg. loss 18.33, avg. ppl 2.05 cum. examples 11036, speed 1193.64 words/sec, time elapsed 1251.79 sec\n",
            "epoch 12, iter 4700, avg. loss 12.25, avg. ppl 1.75 cum. examples 11196, speed 1239.01 words/sec, time elapsed 1254.62 sec\n",
            "epoch 12, iter 4710, avg. loss 7.03, avg. ppl 1.43 cum. examples 11356, speed 1454.14 words/sec, time elapsed 1256.79 sec\n",
            "epoch 12, iter 4720, avg. loss 11.17, avg. ppl 1.65 cum. examples 11516, speed 1402.35 words/sec, time elapsed 1259.32 sec\n",
            "epoch 12, iter 4730, avg. loss 17.67, avg. ppl 2.03 cum. examples 11676, speed 1254.92 words/sec, time elapsed 1262.50 sec\n",
            "epoch 12, iter 4740, avg. loss 10.32, avg. ppl 1.63 cum. examples 11836, speed 1449.46 words/sec, time elapsed 1264.84 sec\n",
            "epoch 12, iter 4750, avg. loss 8.04, avg. ppl 1.48 cum. examples 11996, speed 1420.37 words/sec, time elapsed 1267.13 sec\n",
            "epoch 12, iter 4760, avg. loss 10.38, avg. ppl 1.65 cum. examples 12156, speed 1513.11 words/sec, time elapsed 1269.33 sec\n",
            "epoch 12, iter 4770, avg. loss 15.50, avg. ppl 1.91 cum. examples 12316, speed 1336.32 words/sec, time elapsed 1272.19 sec\n",
            "epoch 12, iter 4780, avg. loss 13.54, avg. ppl 1.85 cum. examples 12476, speed 1252.40 words/sec, time elapsed 1275.01 sec\n",
            "epoch 12, iter 4790, avg. loss 15.01, avg. ppl 1.89 cum. examples 12636, speed 1275.09 words/sec, time elapsed 1277.97 sec\n",
            "epoch 12, iter 4800, avg. loss 10.30, avg. ppl 1.62 cum. examples 12796, speed 1511.66 words/sec, time elapsed 1280.24 sec\n",
            "epoch 12, iter 4800, cum. loss 12.70, cum. ppl 1.78 cum. examples 12796\n",
            "begin validation ...\n",
            "validation: iter 4800, dev. ppl 445.820379\n",
            "hit patience 5\n",
            "hit #1 trial\n",
            "load previously best model and decay learning rate to 0.000500\n",
            "restore parameters of the optimizers\n",
            "epoch 12, iter 4810, avg. loss 82.22, avg. ppl 42.75 cum. examples 160, speed 447.69 words/sec, time elapsed 1288.07 sec\n",
            "epoch 13, iter 4820, avg. loss 82.33, avg. ppl 40.57 cum. examples 318, speed 1567.35 words/sec, time elapsed 1290.31 sec\n",
            "epoch 13, iter 4830, avg. loss 88.79, avg. ppl 41.03 cum. examples 478, speed 1344.81 words/sec, time elapsed 1293.15 sec\n",
            "epoch 13, iter 4840, avg. loss 83.83, avg. ppl 37.72 cum. examples 638, speed 1286.41 words/sec, time elapsed 1296.03 sec\n",
            "epoch 13, iter 4850, avg. loss 76.74, avg. ppl 35.71 cum. examples 798, speed 1369.02 words/sec, time elapsed 1298.53 sec\n",
            "epoch 13, iter 4860, avg. loss 81.55, avg. ppl 39.64 cum. examples 958, speed 1511.32 words/sec, time elapsed 1300.88 sec\n",
            "epoch 13, iter 4870, avg. loss 83.16, avg. ppl 38.81 cum. examples 1118, speed 1465.31 words/sec, time elapsed 1303.36 sec\n",
            "epoch 13, iter 4880, avg. loss 84.30, avg. ppl 39.30 cum. examples 1278, speed 1164.72 words/sec, time elapsed 1306.52 sec\n",
            "epoch 13, iter 4890, avg. loss 84.31, avg. ppl 38.40 cum. examples 1438, speed 1345.29 words/sec, time elapsed 1309.27 sec\n",
            "epoch 13, iter 4900, avg. loss 72.24, avg. ppl 32.85 cum. examples 1598, speed 1321.18 words/sec, time elapsed 1311.77 sec\n",
            "epoch 13, iter 4910, avg. loss 85.71, avg. ppl 42.95 cum. examples 1758, speed 1340.31 words/sec, time elapsed 1314.49 sec\n",
            "epoch 13, iter 4920, avg. loss 87.64, avg. ppl 39.50 cum. examples 1918, speed 1462.49 words/sec, time elapsed 1317.10 sec\n",
            "epoch 13, iter 4930, avg. loss 76.83, avg. ppl 34.56 cum. examples 2078, speed 1371.20 words/sec, time elapsed 1319.63 sec\n",
            "epoch 13, iter 4940, avg. loss 84.26, avg. ppl 39.50 cum. examples 2238, speed 1299.29 words/sec, time elapsed 1322.45 sec\n",
            "epoch 13, iter 4950, avg. loss 83.28, avg. ppl 37.56 cum. examples 2398, speed 1344.91 words/sec, time elapsed 1325.19 sec\n",
            "epoch 13, iter 4960, avg. loss 71.51, avg. ppl 33.18 cum. examples 2558, speed 1557.55 words/sec, time elapsed 1327.28 sec\n",
            "epoch 13, iter 4970, avg. loss 70.77, avg. ppl 32.73 cum. examples 2718, speed 1598.58 words/sec, time elapsed 1329.31 sec\n",
            "epoch 13, iter 4980, avg. loss 78.23, avg. ppl 33.25 cum. examples 2878, speed 1474.87 words/sec, time elapsed 1331.74 sec\n",
            "epoch 13, iter 4990, avg. loss 79.19, avg. ppl 35.31 cum. examples 3038, speed 1214.10 words/sec, time elapsed 1334.66 sec\n",
            "epoch 13, iter 5000, avg. loss 81.77, avg. ppl 39.05 cum. examples 3198, speed 1330.52 words/sec, time elapsed 1337.35 sec\n",
            "epoch 13, iter 5010, avg. loss 74.67, avg. ppl 33.13 cum. examples 3358, speed 1347.51 words/sec, time elapsed 1339.88 sec\n",
            "epoch 13, iter 5020, avg. loss 79.71, avg. ppl 33.63 cum. examples 3518, speed 1114.24 words/sec, time elapsed 1343.14 sec\n",
            "epoch 13, iter 5030, avg. loss 76.07, avg. ppl 32.73 cum. examples 3678, speed 1595.52 words/sec, time elapsed 1345.32 sec\n",
            "epoch 13, iter 5040, avg. loss 78.34, avg. ppl 34.64 cum. examples 3838, speed 1199.82 words/sec, time elapsed 1348.27 sec\n",
            "epoch 13, iter 5050, avg. loss 78.63, avg. ppl 34.37 cum. examples 3998, speed 1405.95 words/sec, time elapsed 1350.80 sec\n",
            "epoch 13, iter 5060, avg. loss 71.46, avg. ppl 31.40 cum. examples 4158, speed 1648.48 words/sec, time elapsed 1352.81 sec\n",
            "epoch 13, iter 5070, avg. loss 72.00, avg. ppl 33.27 cum. examples 4318, speed 1385.58 words/sec, time elapsed 1355.18 sec\n",
            "epoch 13, iter 5080, avg. loss 76.25, avg. ppl 34.19 cum. examples 4478, speed 1466.61 words/sec, time elapsed 1357.54 sec\n",
            "epoch 13, iter 5090, avg. loss 86.50, avg. ppl 39.52 cum. examples 4638, speed 1338.44 words/sec, time elapsed 1360.35 sec\n",
            "epoch 13, iter 5100, avg. loss 77.08, avg. ppl 33.57 cum. examples 4798, speed 1253.71 words/sec, time elapsed 1363.15 sec\n",
            "epoch 13, iter 5110, avg. loss 75.39, avg. ppl 33.71 cum. examples 4958, speed 1486.92 words/sec, time elapsed 1365.46 sec\n",
            "epoch 13, iter 5120, avg. loss 80.46, avg. ppl 35.74 cum. examples 5118, speed 1567.36 words/sec, time elapsed 1367.75 sec\n",
            "epoch 13, iter 5130, avg. loss 79.96, avg. ppl 37.37 cum. examples 5278, speed 1490.45 words/sec, time elapsed 1370.12 sec\n",
            "epoch 13, iter 5140, avg. loss 70.94, avg. ppl 30.91 cum. examples 5438, speed 1302.30 words/sec, time elapsed 1372.67 sec\n",
            "epoch 13, iter 5150, avg. loss 82.14, avg. ppl 37.66 cum. examples 5598, speed 1392.05 words/sec, time elapsed 1375.27 sec\n",
            "epoch 13, iter 5160, avg. loss 78.70, avg. ppl 37.75 cum. examples 5758, speed 1477.32 words/sec, time elapsed 1377.61 sec\n",
            "epoch 13, iter 5170, avg. loss 88.58, avg. ppl 39.59 cum. examples 5918, speed 1307.01 words/sec, time elapsed 1380.56 sec\n",
            "epoch 13, iter 5180, avg. loss 77.29, avg. ppl 35.07 cum. examples 6078, speed 1417.65 words/sec, time elapsed 1383.01 sec\n",
            "epoch 13, iter 5190, avg. loss 85.70, avg. ppl 37.95 cum. examples 6238, speed 1134.35 words/sec, time elapsed 1386.34 sec\n",
            "epoch 13, iter 5200, avg. loss 72.98, avg. ppl 32.17 cum. examples 6398, speed 1324.73 words/sec, time elapsed 1388.88 sec\n",
            "epoch 13, iter 5210, avg. loss 66.31, avg. ppl 30.98 cum. examples 6558, speed 1473.57 words/sec, time elapsed 1390.98 sec\n",
            "epoch 14, iter 5220, avg. loss 73.95, avg. ppl 26.02 cum. examples 6716, speed 1167.91 words/sec, time elapsed 1394.05 sec\n",
            "epoch 14, iter 5230, avg. loss 68.46, avg. ppl 20.67 cum. examples 6876, speed 1409.20 words/sec, time elapsed 1396.61 sec\n",
            "epoch 14, iter 5240, avg. loss 74.81, avg. ppl 24.88 cum. examples 7036, speed 1320.94 words/sec, time elapsed 1399.43 sec\n",
            "epoch 14, iter 5250, avg. loss 65.85, avg. ppl 20.43 cum. examples 7196, speed 1352.12 words/sec, time elapsed 1402.01 sec\n",
            "epoch 14, iter 5260, avg. loss 68.12, avg. ppl 22.08 cum. examples 7356, speed 1332.52 words/sec, time elapsed 1404.66 sec\n",
            "epoch 14, iter 5270, avg. loss 74.93, avg. ppl 23.69 cum. examples 7516, speed 1351.85 words/sec, time elapsed 1407.46 sec\n",
            "epoch 14, iter 5280, avg. loss 67.29, avg. ppl 20.87 cum. examples 7676, speed 1417.26 words/sec, time elapsed 1409.96 sec\n",
            "epoch 14, iter 5290, avg. loss 62.32, avg. ppl 20.46 cum. examples 7836, speed 1299.99 words/sec, time elapsed 1412.50 sec\n",
            "epoch 14, iter 5300, avg. loss 68.90, avg. ppl 21.30 cum. examples 7996, speed 1511.31 words/sec, time elapsed 1414.89 sec\n",
            "epoch 14, iter 5310, avg. loss 65.26, avg. ppl 20.47 cum. examples 8156, speed 1425.17 words/sec, time elapsed 1417.31 sec\n",
            "epoch 14, iter 5320, avg. loss 68.59, avg. ppl 23.64 cum. examples 8316, speed 1457.60 words/sec, time elapsed 1419.69 sec\n",
            "epoch 14, iter 5330, avg. loss 82.02, avg. ppl 26.90 cum. examples 8476, speed 1233.41 words/sec, time elapsed 1422.92 sec\n",
            "epoch 14, iter 5340, avg. loss 67.85, avg. ppl 22.17 cum. examples 8636, speed 1307.40 words/sec, time elapsed 1425.60 sec\n",
            "epoch 14, iter 5350, avg. loss 78.87, avg. ppl 25.34 cum. examples 8796, speed 1359.61 words/sec, time elapsed 1428.48 sec\n",
            "epoch 14, iter 5360, avg. loss 61.77, avg. ppl 21.53 cum. examples 8956, speed 1342.16 words/sec, time elapsed 1430.87 sec\n",
            "epoch 14, iter 5370, avg. loss 75.42, avg. ppl 25.09 cum. examples 9116, speed 1354.90 words/sec, time elapsed 1433.64 sec\n",
            "epoch 14, iter 5380, avg. loss 70.91, avg. ppl 23.48 cum. examples 9276, speed 1307.44 words/sec, time elapsed 1436.39 sec\n",
            "epoch 14, iter 5390, avg. loss 75.29, avg. ppl 25.60 cum. examples 9436, speed 1662.05 words/sec, time elapsed 1438.62 sec\n",
            "epoch 14, iter 5400, avg. loss 69.27, avg. ppl 22.49 cum. examples 9596, speed 1287.61 words/sec, time elapsed 1441.39 sec\n",
            "epoch 14, iter 5410, avg. loss 64.31, avg. ppl 19.17 cum. examples 9756, speed 1372.09 words/sec, time elapsed 1443.93 sec\n",
            "epoch 14, iter 5420, avg. loss 69.20, avg. ppl 23.80 cum. examples 9916, speed 1371.29 words/sec, time elapsed 1446.48 sec\n",
            "epoch 14, iter 5430, avg. loss 69.40, avg. ppl 23.70 cum. examples 10076, speed 1237.89 words/sec, time elapsed 1449.31 sec\n",
            "epoch 14, iter 5440, avg. loss 62.67, avg. ppl 22.47 cum. examples 10236, speed 1451.69 words/sec, time elapsed 1451.53 sec\n",
            "epoch 14, iter 5450, avg. loss 69.31, avg. ppl 21.95 cum. examples 10396, speed 1265.14 words/sec, time elapsed 1454.37 sec\n",
            "epoch 14, iter 5460, avg. loss 78.55, avg. ppl 26.03 cum. examples 10556, speed 1369.91 words/sec, time elapsed 1457.18 sec\n",
            "epoch 14, iter 5470, avg. loss 70.88, avg. ppl 22.88 cum. examples 10716, speed 1385.73 words/sec, time elapsed 1459.80 sec\n",
            "epoch 14, iter 5480, avg. loss 62.43, avg. ppl 21.39 cum. examples 10876, speed 1381.63 words/sec, time elapsed 1462.16 sec\n",
            "epoch 14, iter 5490, avg. loss 73.90, avg. ppl 23.40 cum. examples 11036, speed 1370.10 words/sec, time elapsed 1464.89 sec\n",
            "epoch 14, iter 5500, avg. loss 67.93, avg. ppl 21.87 cum. examples 11196, speed 1233.83 words/sec, time elapsed 1467.75 sec\n",
            "epoch 14, iter 5510, avg. loss 65.10, avg. ppl 21.44 cum. examples 11356, speed 1390.36 words/sec, time elapsed 1470.19 sec\n",
            "epoch 14, iter 5520, avg. loss 63.08, avg. ppl 20.83 cum. examples 11516, speed 1391.49 words/sec, time elapsed 1472.58 sec\n",
            "epoch 14, iter 5530, avg. loss 66.03, avg. ppl 22.78 cum. examples 11676, speed 1676.39 words/sec, time elapsed 1474.60 sec\n",
            "epoch 14, iter 5540, avg. loss 68.56, avg. ppl 24.26 cum. examples 11836, speed 1350.80 words/sec, time elapsed 1477.14 sec\n",
            "epoch 14, iter 5550, avg. loss 70.30, avg. ppl 22.84 cum. examples 11996, speed 1210.57 words/sec, time elapsed 1480.11 sec\n",
            "epoch 14, iter 5560, avg. loss 60.30, avg. ppl 20.96 cum. examples 12156, speed 1672.55 words/sec, time elapsed 1482.01 sec\n",
            "epoch 14, iter 5570, avg. loss 71.32, avg. ppl 23.18 cum. examples 12316, speed 1404.83 words/sec, time elapsed 1484.59 sec\n",
            "epoch 14, iter 5580, avg. loss 70.47, avg. ppl 24.00 cum. examples 12476, speed 1502.59 words/sec, time elapsed 1486.96 sec\n",
            "epoch 14, iter 5590, avg. loss 70.94, avg. ppl 24.58 cum. examples 12636, speed 1636.11 words/sec, time elapsed 1489.12 sec\n",
            "epoch 14, iter 5600, avg. loss 66.34, avg. ppl 24.35 cum. examples 12796, speed 1418.07 words/sec, time elapsed 1491.47 sec\n",
            "epoch 14, iter 5600, cum. loss 74.36, cum. ppl 28.92 cum. examples 12796\n",
            "begin validation ...\n",
            "validation: iter 5600, dev. ppl 61.990932\n",
            "hit patience 1\n",
            "epoch 14, iter 5610, avg. loss 73.50, avg. ppl 24.72 cum. examples 160, speed 471.89 words/sec, time elapsed 1499.24 sec\n",
            "epoch 15, iter 5620, avg. loss 58.20, avg. ppl 16.13 cum. examples 318, speed 1451.60 words/sec, time elapsed 1501.51 sec\n",
            "epoch 15, iter 5630, avg. loss 47.53, avg. ppl 12.14 cum. examples 478, speed 1446.79 words/sec, time elapsed 1503.62 sec\n",
            "epoch 15, iter 5640, avg. loss 52.00, avg. ppl 11.99 cum. examples 638, speed 1180.36 words/sec, time elapsed 1506.46 sec\n",
            "epoch 15, iter 5650, avg. loss 56.01, avg. ppl 14.14 cum. examples 798, speed 1403.61 words/sec, time elapsed 1508.87 sec\n",
            "epoch 15, iter 5660, avg. loss 61.01, avg. ppl 14.61 cum. examples 958, speed 1390.82 words/sec, time elapsed 1511.48 sec\n",
            "epoch 15, iter 5670, avg. loss 58.29, avg. ppl 13.26 cum. examples 1118, speed 1462.96 words/sec, time elapsed 1513.95 sec\n",
            "epoch 15, iter 5680, avg. loss 55.10, avg. ppl 13.10 cum. examples 1278, speed 1424.34 words/sec, time elapsed 1516.36 sec\n",
            "epoch 15, iter 5690, avg. loss 57.21, avg. ppl 13.55 cum. examples 1438, speed 1531.39 words/sec, time elapsed 1518.65 sec\n",
            "epoch 15, iter 5700, avg. loss 57.08, avg. ppl 14.09 cum. examples 1598, speed 1499.10 words/sec, time elapsed 1520.95 sec\n",
            "epoch 15, iter 5710, avg. loss 57.14, avg. ppl 13.72 cum. examples 1758, speed 1291.46 words/sec, time elapsed 1523.66 sec\n",
            "epoch 15, iter 5720, avg. loss 62.06, avg. ppl 14.89 cum. examples 1918, speed 1532.49 words/sec, time elapsed 1526.06 sec\n",
            "epoch 15, iter 5730, avg. loss 56.97, avg. ppl 14.21 cum. examples 2078, speed 1195.88 words/sec, time elapsed 1528.93 sec\n",
            "epoch 15, iter 5740, avg. loss 60.00, avg. ppl 15.12 cum. examples 2238, speed 1396.50 words/sec, time elapsed 1531.46 sec\n",
            "epoch 15, iter 5750, avg. loss 61.46, avg. ppl 15.43 cum. examples 2398, speed 1493.88 words/sec, time elapsed 1533.87 sec\n",
            "epoch 15, iter 5760, avg. loss 58.47, avg. ppl 14.51 cum. examples 2558, speed 1357.51 words/sec, time elapsed 1536.44 sec\n",
            "epoch 15, iter 5770, avg. loss 67.58, avg. ppl 17.51 cum. examples 2718, speed 1417.77 words/sec, time elapsed 1539.11 sec\n",
            "epoch 15, iter 5780, avg. loss 73.38, avg. ppl 18.39 cum. examples 2878, speed 1440.76 words/sec, time elapsed 1541.91 sec\n",
            "epoch 15, iter 5790, avg. loss 59.25, avg. ppl 14.42 cum. examples 3038, speed 1566.74 words/sec, time elapsed 1544.17 sec\n",
            "epoch 15, iter 5800, avg. loss 59.02, avg. ppl 15.41 cum. examples 3198, speed 1184.37 words/sec, time elapsed 1547.09 sec\n",
            "epoch 15, iter 5810, avg. loss 57.70, avg. ppl 13.80 cum. examples 3358, speed 1438.22 words/sec, time elapsed 1549.53 sec\n",
            "epoch 15, iter 5820, avg. loss 58.39, avg. ppl 14.64 cum. examples 3518, speed 1227.19 words/sec, time elapsed 1552.37 sec\n",
            "epoch 15, iter 5830, avg. loss 60.28, avg. ppl 15.07 cum. examples 3678, speed 1489.52 words/sec, time elapsed 1554.76 sec\n",
            "epoch 15, iter 5840, avg. loss 55.54, avg. ppl 14.10 cum. examples 3838, speed 1427.61 words/sec, time elapsed 1557.11 sec\n",
            "epoch 15, iter 5850, avg. loss 57.56, avg. ppl 14.41 cum. examples 3998, speed 1208.82 words/sec, time elapsed 1559.97 sec\n",
            "epoch 15, iter 5860, avg. loss 57.62, avg. ppl 14.22 cum. examples 4158, speed 1462.09 words/sec, time elapsed 1562.34 sec\n",
            "epoch 15, iter 5870, avg. loss 59.88, avg. ppl 15.92 cum. examples 4318, speed 1435.76 words/sec, time elapsed 1564.75 sec\n",
            "epoch 15, iter 5880, avg. loss 69.43, avg. ppl 17.27 cum. examples 4478, speed 1132.25 words/sec, time elapsed 1568.20 sec\n",
            "epoch 15, iter 5890, avg. loss 62.17, avg. ppl 15.92 cum. examples 4638, speed 1518.69 words/sec, time elapsed 1570.56 sec\n",
            "epoch 15, iter 5900, avg. loss 57.33, avg. ppl 13.12 cum. examples 4798, speed 1407.19 words/sec, time elapsed 1573.10 sec\n",
            "epoch 15, iter 5910, avg. loss 68.56, avg. ppl 17.33 cum. examples 4958, speed 1355.72 words/sec, time elapsed 1575.93 sec\n",
            "epoch 15, iter 5920, avg. loss 66.97, avg. ppl 17.25 cum. examples 5118, speed 1332.47 words/sec, time elapsed 1578.76 sec\n",
            "epoch 15, iter 5930, avg. loss 65.21, avg. ppl 16.61 cum. examples 5278, speed 1544.69 words/sec, time elapsed 1581.16 sec\n",
            "epoch 15, iter 5940, avg. loss 70.31, avg. ppl 17.76 cum. examples 5438, speed 1273.68 words/sec, time elapsed 1584.23 sec\n",
            "epoch 15, iter 5950, avg. loss 49.52, avg. ppl 12.37 cum. examples 5598, speed 1459.03 words/sec, time elapsed 1586.39 sec\n",
            "epoch 15, iter 5960, avg. loss 59.71, avg. ppl 14.23 cum. examples 5758, speed 1478.10 words/sec, time elapsed 1588.82 sec\n",
            "epoch 15, iter 5970, avg. loss 66.68, avg. ppl 15.85 cum. examples 5918, speed 1176.96 words/sec, time elapsed 1592.10 sec\n",
            "epoch 15, iter 5980, avg. loss 59.87, avg. ppl 14.86 cum. examples 6078, speed 1479.09 words/sec, time elapsed 1594.50 sec\n",
            "epoch 15, iter 5990, avg. loss 49.76, avg. ppl 12.52 cum. examples 6238, speed 1522.04 words/sec, time elapsed 1596.57 sec\n",
            "epoch 15, iter 6000, avg. loss 56.70, avg. ppl 15.22 cum. examples 6398, speed 1393.47 words/sec, time elapsed 1598.97 sec\n",
            "epoch 15, iter 6010, avg. loss 56.81, avg. ppl 13.98 cum. examples 6558, speed 1424.34 words/sec, time elapsed 1601.38 sec\n",
            "epoch 16, iter 6020, avg. loss 46.63, avg. ppl 9.60 cum. examples 6716, speed 1514.37 words/sec, time elapsed 1603.54 sec\n",
            "epoch 16, iter 6030, avg. loss 41.88, avg. ppl 7.69 cum. examples 6876, speed 1601.50 words/sec, time elapsed 1605.59 sec\n",
            "epoch 16, iter 6040, avg. loss 56.03, avg. ppl 10.66 cum. examples 7036, speed 1252.61 words/sec, time elapsed 1608.61 sec\n",
            "epoch 16, iter 6050, avg. loss 45.54, avg. ppl 8.37 cum. examples 7196, speed 1689.31 words/sec, time elapsed 1610.64 sec\n",
            "epoch 16, iter 6060, avg. loss 51.78, avg. ppl 9.60 cum. examples 7356, speed 1529.53 words/sec, time elapsed 1613.04 sec\n",
            "epoch 16, iter 6070, avg. loss 43.70, avg. ppl 8.18 cum. examples 7516, speed 1519.15 words/sec, time elapsed 1615.23 sec\n",
            "epoch 16, iter 6080, avg. loss 51.13, avg. ppl 9.38 cum. examples 7676, speed 1462.84 words/sec, time elapsed 1617.72 sec\n",
            "epoch 16, iter 6090, avg. loss 49.41, avg. ppl 9.34 cum. examples 7836, speed 1379.04 words/sec, time elapsed 1620.29 sec\n",
            "epoch 16, iter 6100, avg. loss 47.15, avg. ppl 9.10 cum. examples 7996, speed 1324.41 words/sec, time elapsed 1622.87 sec\n",
            "epoch 16, iter 6110, avg. loss 49.61, avg. ppl 9.13 cum. examples 8156, speed 1246.15 words/sec, time elapsed 1625.75 sec\n",
            "epoch 16, iter 6120, avg. loss 55.14, avg. ppl 9.86 cum. examples 8316, speed 1347.74 words/sec, time elapsed 1628.61 sec\n",
            "epoch 16, iter 6130, avg. loss 54.38, avg. ppl 10.05 cum. examples 8476, speed 1299.40 words/sec, time elapsed 1631.51 sec\n",
            "epoch 16, iter 6140, avg. loss 52.94, avg. ppl 9.94 cum. examples 8636, speed 1075.09 words/sec, time elapsed 1634.94 sec\n",
            "epoch 16, iter 6150, avg. loss 42.28, avg. ppl 7.93 cum. examples 8796, speed 1435.19 words/sec, time elapsed 1637.22 sec\n",
            "epoch 16, iter 6160, avg. loss 46.33, avg. ppl 9.03 cum. examples 8956, speed 1612.71 words/sec, time elapsed 1639.31 sec\n",
            "epoch 16, iter 6170, avg. loss 46.88, avg. ppl 9.34 cum. examples 9116, speed 1572.98 words/sec, time elapsed 1641.44 sec\n",
            "epoch 16, iter 6180, avg. loss 50.49, avg. ppl 9.68 cum. examples 9276, speed 1481.50 words/sec, time elapsed 1643.85 sec\n",
            "epoch 16, iter 6190, avg. loss 49.43, avg. ppl 9.80 cum. examples 9436, speed 1428.54 words/sec, time elapsed 1646.27 sec\n",
            "epoch 16, iter 6200, avg. loss 49.33, avg. ppl 9.92 cum. examples 9596, speed 1414.70 words/sec, time elapsed 1648.70 sec\n",
            "epoch 16, iter 6210, avg. loss 46.86, avg. ppl 9.40 cum. examples 9756, speed 1363.34 words/sec, time elapsed 1651.16 sec\n",
            "epoch 16, iter 6220, avg. loss 50.89, avg. ppl 10.04 cum. examples 9916, speed 1446.04 words/sec, time elapsed 1653.60 sec\n",
            "epoch 16, iter 6230, avg. loss 49.87, avg. ppl 9.95 cum. examples 10076, speed 1395.74 words/sec, time elapsed 1656.09 sec\n",
            "epoch 16, iter 6240, avg. loss 49.94, avg. ppl 9.74 cum. examples 10236, speed 1251.89 words/sec, time elapsed 1658.89 sec\n",
            "epoch 16, iter 6250, avg. loss 46.41, avg. ppl 8.78 cum. examples 10396, speed 1486.15 words/sec, time elapsed 1661.19 sec\n",
            "epoch 16, iter 6260, avg. loss 61.59, avg. ppl 12.87 cum. examples 10556, speed 1286.07 words/sec, time elapsed 1664.19 sec\n",
            "epoch 16, iter 6270, avg. loss 43.25, avg. ppl 8.52 cum. examples 10716, speed 1389.23 words/sec, time elapsed 1666.52 sec\n",
            "epoch 16, iter 6280, avg. loss 57.66, avg. ppl 11.14 cum. examples 10876, speed 1349.88 words/sec, time elapsed 1669.35 sec\n",
            "epoch 16, iter 6290, avg. loss 57.79, avg. ppl 11.62 cum. examples 11036, speed 1227.59 words/sec, time elapsed 1672.42 sec\n",
            "epoch 16, iter 6300, avg. loss 58.93, avg. ppl 11.51 cum. examples 11196, speed 1249.35 words/sec, time elapsed 1675.51 sec\n",
            "epoch 16, iter 6310, avg. loss 52.62, avg. ppl 10.27 cum. examples 11356, speed 1504.62 words/sec, time elapsed 1677.91 sec\n",
            "epoch 16, iter 6320, avg. loss 56.17, avg. ppl 11.60 cum. examples 11516, speed 1198.33 words/sec, time elapsed 1680.97 sec\n",
            "epoch 16, iter 6330, avg. loss 49.07, avg. ppl 9.57 cum. examples 11676, speed 1382.74 words/sec, time elapsed 1683.49 sec\n",
            "epoch 16, iter 6340, avg. loss 48.43, avg. ppl 9.74 cum. examples 11836, speed 1464.86 words/sec, time elapsed 1685.81 sec\n",
            "epoch 16, iter 6350, avg. loss 50.29, avg. ppl 10.13 cum. examples 11996, speed 1364.49 words/sec, time elapsed 1688.36 sec\n",
            "epoch 16, iter 6360, avg. loss 45.97, avg. ppl 9.53 cum. examples 12156, speed 1299.57 words/sec, time elapsed 1690.87 sec\n",
            "epoch 16, iter 6370, avg. loss 51.47, avg. ppl 10.19 cum. examples 12316, speed 1235.02 words/sec, time elapsed 1693.74 sec\n",
            "epoch 16, iter 6380, avg. loss 46.54, avg. ppl 9.68 cum. examples 12476, speed 1287.01 words/sec, time elapsed 1696.29 sec\n",
            "epoch 16, iter 6390, avg. loss 57.74, avg. ppl 12.14 cum. examples 12636, speed 1296.71 words/sec, time elapsed 1699.14 sec\n",
            "epoch 16, iter 6400, avg. loss 55.78, avg. ppl 11.02 cum. examples 12796, speed 1274.30 words/sec, time elapsed 1702.06 sec\n",
            "epoch 16, iter 6400, cum. loss 55.31, cum. ppl 12.21 cum. examples 12796\n",
            "begin validation ...\n",
            "validation: iter 6400, dev. ppl 87.796557\n",
            "hit patience 2\n",
            "epoch 16, iter 6410, avg. loss 49.52, avg. ppl 9.56 cum. examples 160, speed 409.22 words/sec, time elapsed 1710.64 sec\n",
            "epoch 17, iter 6420, avg. loss 48.78, avg. ppl 9.00 cum. examples 318, speed 1323.77 words/sec, time elapsed 1713.29 sec\n",
            "epoch 17, iter 6430, avg. loss 37.83, avg. ppl 5.98 cum. examples 478, speed 1316.90 words/sec, time elapsed 1715.86 sec\n",
            "epoch 17, iter 6440, avg. loss 37.72, avg. ppl 6.12 cum. examples 638, speed 1272.79 words/sec, time elapsed 1718.48 sec\n",
            "epoch 17, iter 6450, avg. loss 47.93, avg. ppl 7.45 cum. examples 798, speed 1524.16 words/sec, time elapsed 1720.98 sec\n",
            "epoch 17, iter 6460, avg. loss 45.97, avg. ppl 6.95 cum. examples 958, speed 1475.94 words/sec, time elapsed 1723.55 sec\n",
            "epoch 17, iter 6470, avg. loss 34.32, avg. ppl 5.42 cum. examples 1118, speed 1429.02 words/sec, time elapsed 1725.83 sec\n",
            "epoch 17, iter 6480, avg. loss 33.12, avg. ppl 5.44 cum. examples 1278, speed 1362.44 words/sec, time elapsed 1728.12 sec\n",
            "epoch 17, iter 6490, avg. loss 37.00, avg. ppl 5.70 cum. examples 1438, speed 1528.80 words/sec, time elapsed 1730.35 sec\n",
            "epoch 17, iter 6500, avg. loss 37.67, avg. ppl 5.83 cum. examples 1598, speed 1350.48 words/sec, time elapsed 1732.88 sec\n",
            "epoch 17, iter 6510, avg. loss 35.41, avg. ppl 5.37 cum. examples 1758, speed 1604.86 words/sec, time elapsed 1734.98 sec\n",
            "epoch 17, iter 6520, avg. loss 38.99, avg. ppl 6.03 cum. examples 1918, speed 1411.92 words/sec, time elapsed 1737.44 sec\n",
            "epoch 17, iter 6530, avg. loss 38.52, avg. ppl 6.14 cum. examples 2078, speed 1301.55 words/sec, time elapsed 1740.05 sec\n",
            "epoch 17, iter 6540, avg. loss 34.23, avg. ppl 5.53 cum. examples 2238, speed 1456.80 words/sec, time elapsed 1742.25 sec\n",
            "epoch 17, iter 6550, avg. loss 51.14, avg. ppl 8.08 cum. examples 2398, speed 1164.95 words/sec, time elapsed 1745.61 sec\n",
            "epoch 17, iter 6560, avg. loss 41.45, avg. ppl 6.46 cum. examples 2558, speed 1362.65 words/sec, time elapsed 1748.22 sec\n",
            "epoch 17, iter 6570, avg. loss 38.12, avg. ppl 6.01 cum. examples 2718, speed 1530.60 words/sec, time elapsed 1750.44 sec\n",
            "epoch 17, iter 6580, avg. loss 39.58, avg. ppl 6.30 cum. examples 2878, speed 1313.84 words/sec, time elapsed 1753.06 sec\n",
            "epoch 17, iter 6590, avg. loss 43.57, avg. ppl 6.92 cum. examples 3038, speed 1312.96 words/sec, time elapsed 1755.80 sec\n",
            "epoch 17, iter 6600, avg. loss 44.78, avg. ppl 7.59 cum. examples 3198, speed 1399.36 words/sec, time elapsed 1758.33 sec\n",
            "epoch 17, iter 6610, avg. loss 34.79, avg. ppl 5.68 cum. examples 3358, speed 1384.19 words/sec, time elapsed 1760.64 sec\n",
            "epoch 17, iter 6620, avg. loss 38.66, avg. ppl 6.08 cum. examples 3518, speed 1379.32 words/sec, time elapsed 1763.13 sec\n",
            "epoch 17, iter 6630, avg. loss 38.58, avg. ppl 6.07 cum. examples 3678, speed 1903.08 words/sec, time elapsed 1764.93 sec\n",
            "epoch 17, iter 6640, avg. loss 43.97, avg. ppl 7.15 cum. examples 3838, speed 1369.75 words/sec, time elapsed 1767.54 sec\n",
            "epoch 17, iter 6650, avg. loss 45.78, avg. ppl 7.42 cum. examples 3998, speed 1282.29 words/sec, time elapsed 1770.39 sec\n",
            "epoch 17, iter 6660, avg. loss 42.08, avg. ppl 7.45 cum. examples 4158, speed 1445.16 words/sec, time elapsed 1772.71 sec\n",
            "epoch 17, iter 6670, avg. loss 38.85, avg. ppl 6.21 cum. examples 4318, speed 1436.74 words/sec, time elapsed 1775.08 sec\n",
            "epoch 17, iter 6680, avg. loss 42.91, avg. ppl 6.58 cum. examples 4478, speed 1399.60 words/sec, time elapsed 1777.68 sec\n",
            "epoch 17, iter 6690, avg. loss 48.71, avg. ppl 7.58 cum. examples 4638, speed 1320.29 words/sec, time elapsed 1780.60 sec\n",
            "epoch 17, iter 6700, avg. loss 48.26, avg. ppl 7.83 cum. examples 4798, speed 1254.32 words/sec, time elapsed 1783.59 sec\n",
            "epoch 17, iter 6710, avg. loss 39.59, avg. ppl 6.29 cum. examples 4958, speed 1534.83 words/sec, time elapsed 1785.83 sec\n",
            "epoch 17, iter 6720, avg. loss 40.18, avg. ppl 6.40 cum. examples 5118, speed 1485.34 words/sec, time elapsed 1788.16 sec\n",
            "epoch 17, iter 6730, avg. loss 51.60, avg. ppl 8.15 cum. examples 5278, speed 1130.94 words/sec, time elapsed 1791.64 sec\n",
            "epoch 17, iter 6740, avg. loss 44.02, avg. ppl 6.97 cum. examples 5438, speed 1342.96 words/sec, time elapsed 1794.35 sec\n",
            "epoch 17, iter 6750, avg. loss 49.86, avg. ppl 7.93 cum. examples 5598, speed 1227.26 words/sec, time elapsed 1797.48 sec\n",
            "epoch 17, iter 6760, avg. loss 42.24, avg. ppl 6.76 cum. examples 5758, speed 1477.20 words/sec, time elapsed 1799.88 sec\n",
            "epoch 17, iter 6770, avg. loss 45.38, avg. ppl 7.47 cum. examples 5918, speed 1358.66 words/sec, time elapsed 1802.54 sec\n",
            "epoch 17, iter 6780, avg. loss 49.65, avg. ppl 8.13 cum. examples 6078, speed 1357.84 words/sec, time elapsed 1805.33 sec\n",
            "epoch 17, iter 6790, avg. loss 45.40, avg. ppl 7.77 cum. examples 6238, speed 1195.51 words/sec, time elapsed 1808.29 sec\n",
            "epoch 17, iter 6800, avg. loss 49.57, avg. ppl 8.86 cum. examples 6398, speed 1159.75 words/sec, time elapsed 1811.43 sec\n",
            "epoch 17, iter 6810, avg. loss 50.34, avg. ppl 8.41 cum. examples 6558, speed 1348.77 words/sec, time elapsed 1814.23 sec\n",
            "epoch 18, iter 6820, avg. loss 40.47, avg. ppl 6.48 cum. examples 6716, speed 1500.51 words/sec, time elapsed 1816.51 sec\n",
            "epoch 18, iter 6830, avg. loss 30.97, avg. ppl 4.25 cum. examples 6876, speed 1138.76 words/sec, time elapsed 1819.52 sec\n",
            "epoch 18, iter 6840, avg. loss 31.85, avg. ppl 4.56 cum. examples 7036, speed 1217.67 words/sec, time elapsed 1822.28 sec\n",
            "epoch 18, iter 6850, avg. loss 40.62, avg. ppl 5.73 cum. examples 7196, speed 1144.62 words/sec, time elapsed 1825.53 sec\n",
            "epoch 18, iter 6860, avg. loss 31.54, avg. ppl 4.40 cum. examples 7356, speed 1595.15 words/sec, time elapsed 1827.66 sec\n",
            "epoch 18, iter 6870, avg. loss 34.11, avg. ppl 4.75 cum. examples 7516, speed 1282.92 words/sec, time elapsed 1830.40 sec\n",
            "epoch 18, iter 6880, avg. loss 38.79, avg. ppl 4.93 cum. examples 7676, speed 1502.45 words/sec, time elapsed 1832.98 sec\n",
            "epoch 18, iter 6890, avg. loss 35.44, avg. ppl 4.71 cum. examples 7836, speed 1438.24 words/sec, time elapsed 1835.53 sec\n",
            "epoch 18, iter 6900, avg. loss 37.40, avg. ppl 5.23 cum. examples 7996, speed 1434.86 words/sec, time elapsed 1838.05 sec\n",
            "epoch 18, iter 6910, avg. loss 37.86, avg. ppl 5.17 cum. examples 8156, speed 1391.35 words/sec, time elapsed 1840.70 sec\n",
            "epoch 18, iter 6920, avg. loss 30.81, avg. ppl 4.19 cum. examples 8316, speed 1590.20 words/sec, time elapsed 1842.86 sec\n",
            "epoch 18, iter 6930, avg. loss 38.41, avg. ppl 5.15 cum. examples 8476, speed 1262.09 words/sec, time elapsed 1845.84 sec\n",
            "epoch 18, iter 6940, avg. loss 32.58, avg. ppl 4.59 cum. examples 8636, speed 1344.69 words/sec, time elapsed 1848.38 sec\n",
            "epoch 18, iter 6950, avg. loss 34.04, avg. ppl 4.73 cum. examples 8796, speed 1376.65 words/sec, time elapsed 1850.93 sec\n",
            "epoch 18, iter 6960, avg. loss 32.04, avg. ppl 4.52 cum. examples 8956, speed 1350.29 words/sec, time elapsed 1853.44 sec\n",
            "epoch 18, iter 6970, avg. loss 32.32, avg. ppl 4.47 cum. examples 9116, speed 1372.23 words/sec, time elapsed 1855.96 sec\n",
            "epoch 18, iter 6980, avg. loss 27.91, avg. ppl 4.13 cum. examples 9276, speed 1420.16 words/sec, time elapsed 1858.18 sec\n",
            "epoch 18, iter 6990, avg. loss 32.51, avg. ppl 4.62 cum. examples 9436, speed 1428.89 words/sec, time elapsed 1860.56 sec\n",
            "epoch 18, iter 7000, avg. loss 30.79, avg. ppl 4.32 cum. examples 9596, speed 1369.87 words/sec, time elapsed 1863.01 sec\n",
            "epoch 18, iter 7010, avg. loss 40.46, avg. ppl 5.74 cum. examples 9756, speed 1341.44 words/sec, time elapsed 1865.78 sec\n",
            "epoch 18, iter 7020, avg. loss 42.95, avg. ppl 5.58 cum. examples 9916, speed 1224.87 words/sec, time elapsed 1869.04 sec\n",
            "epoch 18, iter 7030, avg. loss 42.13, avg. ppl 6.07 cum. examples 10076, speed 1430.48 words/sec, time elapsed 1871.65 sec\n",
            "epoch 18, iter 7040, avg. loss 34.49, avg. ppl 4.88 cum. examples 10236, speed 1256.33 words/sec, time elapsed 1874.42 sec\n",
            "epoch 18, iter 7050, avg. loss 28.99, avg. ppl 4.13 cum. examples 10396, speed 1366.90 words/sec, time elapsed 1876.81 sec\n",
            "epoch 18, iter 7060, avg. loss 35.58, avg. ppl 4.87 cum. examples 10556, speed 1367.52 words/sec, time elapsed 1879.44 sec\n",
            "epoch 18, iter 7070, avg. loss 39.77, avg. ppl 5.53 cum. examples 10716, speed 1276.61 words/sec, time elapsed 1882.36 sec\n",
            "epoch 18, iter 7080, avg. loss 43.48, avg. ppl 6.08 cum. examples 10876, speed 1242.03 words/sec, time elapsed 1885.46 sec\n",
            "epoch 18, iter 7090, avg. loss 33.41, avg. ppl 4.69 cum. examples 11036, speed 1401.35 words/sec, time elapsed 1887.93 sec\n",
            "epoch 18, iter 7100, avg. loss 29.63, avg. ppl 4.18 cum. examples 11196, speed 1561.59 words/sec, time elapsed 1890.05 sec\n",
            "epoch 18, iter 7110, avg. loss 36.47, avg. ppl 5.05 cum. examples 11356, speed 1370.14 words/sec, time elapsed 1892.68 sec\n",
            "epoch 18, iter 7120, avg. loss 37.67, avg. ppl 5.62 cum. examples 11516, speed 1272.34 words/sec, time elapsed 1895.42 sec\n",
            "epoch 18, iter 7130, avg. loss 30.76, avg. ppl 4.22 cum. examples 11676, speed 1597.22 words/sec, time elapsed 1897.56 sec\n",
            "epoch 18, iter 7140, avg. loss 33.00, avg. ppl 4.73 cum. examples 11836, speed 1422.50 words/sec, time elapsed 1899.95 sec\n",
            "epoch 18, iter 7150, avg. loss 39.41, avg. ppl 5.46 cum. examples 11996, speed 1236.18 words/sec, time elapsed 1902.95 sec\n",
            "epoch 18, iter 7160, avg. loss 39.61, avg. ppl 5.48 cum. examples 12156, speed 1585.70 words/sec, time elapsed 1905.30 sec\n",
            "epoch 18, iter 7170, avg. loss 31.64, avg. ppl 4.66 cum. examples 12316, speed 1321.19 words/sec, time elapsed 1907.79 sec\n",
            "epoch 18, iter 7180, avg. loss 32.19, avg. ppl 4.79 cum. examples 12476, speed 1300.04 words/sec, time elapsed 1910.32 sec\n",
            "epoch 18, iter 7190, avg. loss 37.10, avg. ppl 5.54 cum. examples 12636, speed 1324.15 words/sec, time elapsed 1912.94 sec\n",
            "epoch 18, iter 7200, avg. loss 39.38, avg. ppl 5.23 cum. examples 12796, speed 1352.19 words/sec, time elapsed 1915.76 sec\n",
            "epoch 18, iter 7200, cum. loss 39.06, cum. ppl 5.85 cum. examples 12796\n",
            "begin validation ...\n",
            "validation: iter 7200, dev. ppl 134.744115\n",
            "hit patience 3\n",
            "epoch 18, iter 7210, avg. loss 34.13, avg. ppl 4.98 cum. examples 160, speed 448.12 words/sec, time elapsed 1923.35 sec\n",
            "epoch 19, iter 7220, avg. loss 36.33, avg. ppl 4.77 cum. examples 318, speed 1545.80 words/sec, time elapsed 1925.72 sec\n",
            "epoch 19, iter 7230, avg. loss 29.46, avg. ppl 3.58 cum. examples 478, speed 1398.22 words/sec, time elapsed 1928.37 sec\n",
            "epoch 19, iter 7240, avg. loss 25.67, avg. ppl 3.33 cum. examples 638, speed 1234.48 words/sec, time elapsed 1931.14 sec\n",
            "epoch 19, iter 7250, avg. loss 26.68, avg. ppl 3.27 cum. examples 798, speed 1520.32 words/sec, time elapsed 1933.51 sec\n",
            "epoch 19, iter 7260, avg. loss 24.47, avg. ppl 3.26 cum. examples 958, speed 1412.11 words/sec, time elapsed 1935.85 sec\n",
            "epoch 19, iter 7270, avg. loss 27.57, avg. ppl 3.46 cum. examples 1118, speed 1473.41 words/sec, time elapsed 1938.26 sec\n",
            "epoch 19, iter 7280, avg. loss 26.21, avg. ppl 3.39 cum. examples 1278, speed 1296.14 words/sec, time elapsed 1940.91 sec\n",
            "epoch 19, iter 7290, avg. loss 26.60, avg. ppl 3.59 cum. examples 1438, speed 1271.43 words/sec, time elapsed 1943.53 sec\n",
            "epoch 19, iter 7300, avg. loss 27.60, avg. ppl 3.57 cum. examples 1598, speed 1355.66 words/sec, time elapsed 1946.09 sec\n",
            "epoch 19, iter 7310, avg. loss 28.57, avg. ppl 3.71 cum. examples 1758, speed 1128.11 words/sec, time elapsed 1949.18 sec\n",
            "epoch 19, iter 7320, avg. loss 29.22, avg. ppl 3.73 cum. examples 1918, speed 1243.44 words/sec, time elapsed 1952.04 sec\n",
            "epoch 19, iter 7330, avg. loss 31.23, avg. ppl 4.01 cum. examples 2078, speed 1300.46 words/sec, time elapsed 1954.80 sec\n",
            "epoch 19, iter 7340, avg. loss 23.00, avg. ppl 3.03 cum. examples 2238, speed 1534.06 words/sec, time elapsed 1956.97 sec\n",
            "epoch 19, iter 7350, avg. loss 27.47, avg. ppl 3.64 cum. examples 2398, speed 1262.74 words/sec, time elapsed 1959.66 sec\n",
            "epoch 19, iter 7360, avg. loss 32.20, avg. ppl 4.08 cum. examples 2558, speed 1247.75 words/sec, time elapsed 1962.60 sec\n",
            "epoch 19, iter 7370, avg. loss 25.80, avg. ppl 3.26 cum. examples 2718, speed 1644.97 words/sec, time elapsed 1964.72 sec\n",
            "epoch 19, iter 7380, avg. loss 26.27, avg. ppl 3.37 cum. examples 2878, speed 1455.05 words/sec, time elapsed 1967.10 sec\n",
            "epoch 19, iter 7390, avg. loss 33.12, avg. ppl 4.13 cum. examples 3038, speed 1311.75 words/sec, time elapsed 1969.95 sec\n",
            "epoch 19, iter 7400, avg. loss 27.99, avg. ppl 3.52 cum. examples 3198, speed 1359.07 words/sec, time elapsed 1972.57 sec\n",
            "epoch 19, iter 7410, avg. loss 23.24, avg. ppl 3.13 cum. examples 3358, speed 1450.72 words/sec, time elapsed 1974.82 sec\n",
            "epoch 19, iter 7420, avg. loss 26.19, avg. ppl 3.34 cum. examples 3518, speed 1318.05 words/sec, time elapsed 1977.46 sec\n",
            "epoch 19, iter 7430, avg. loss 28.40, avg. ppl 3.60 cum. examples 3678, speed 1431.20 words/sec, time elapsed 1979.93 sec\n",
            "epoch 19, iter 7440, avg. loss 31.63, avg. ppl 4.00 cum. examples 3838, speed 1491.70 words/sec, time elapsed 1982.38 sec\n",
            "epoch 19, iter 7450, avg. loss 26.28, avg. ppl 3.38 cum. examples 3998, speed 1576.27 words/sec, time elapsed 1984.57 sec\n",
            "epoch 19, iter 7460, avg. loss 37.16, avg. ppl 4.42 cum. examples 4158, speed 1262.91 words/sec, time elapsed 1987.74 sec\n",
            "epoch 19, iter 7470, avg. loss 26.61, avg. ppl 3.45 cum. examples 4318, speed 1618.18 words/sec, time elapsed 1989.86 sec\n",
            "epoch 19, iter 7480, avg. loss 31.92, avg. ppl 4.08 cum. examples 4478, speed 1205.43 words/sec, time elapsed 1992.87 sec\n",
            "epoch 19, iter 7490, avg. loss 31.96, avg. ppl 4.34 cum. examples 4638, speed 1167.48 words/sec, time elapsed 1995.85 sec\n",
            "epoch 19, iter 7500, avg. loss 33.97, avg. ppl 4.18 cum. examples 4798, speed 1397.34 words/sec, time elapsed 1998.57 sec\n",
            "epoch 19, iter 7510, avg. loss 36.61, avg. ppl 4.58 cum. examples 4958, speed 1344.70 words/sec, time elapsed 2001.44 sec\n",
            "epoch 19, iter 7520, avg. loss 26.77, avg. ppl 3.51 cum. examples 5118, speed 1432.36 words/sec, time elapsed 2003.82 sec\n",
            "epoch 19, iter 7530, avg. loss 36.92, avg. ppl 4.74 cum. examples 5278, speed 1365.56 words/sec, time elapsed 2006.60 sec\n",
            "epoch 19, iter 7540, avg. loss 27.60, avg. ppl 3.59 cum. examples 5438, speed 1427.14 words/sec, time elapsed 2009.02 sec\n",
            "epoch 19, iter 7550, avg. loss 22.32, avg. ppl 3.09 cum. examples 5598, speed 1358.99 words/sec, time elapsed 2011.35 sec\n",
            "epoch 19, iter 7560, avg. loss 31.27, avg. ppl 3.94 cum. examples 5758, speed 1259.73 words/sec, time elapsed 2014.25 sec\n",
            "epoch 19, iter 7570, avg. loss 33.79, avg. ppl 4.27 cum. examples 5918, speed 1191.36 words/sec, time elapsed 2017.37 sec\n",
            "epoch 19, iter 7580, avg. loss 29.60, avg. ppl 3.66 cum. examples 6078, speed 1396.97 words/sec, time elapsed 2019.98 sec\n",
            "epoch 19, iter 7590, avg. loss 32.91, avg. ppl 4.25 cum. examples 6238, speed 1235.99 words/sec, time elapsed 2022.93 sec\n",
            "epoch 19, iter 7600, avg. loss 28.84, avg. ppl 3.73 cum. examples 6398, speed 1426.07 words/sec, time elapsed 2025.38 sec\n",
            "epoch 19, iter 7610, avg. loss 29.38, avg. ppl 4.05 cum. examples 6558, speed 1236.16 words/sec, time elapsed 2028.10 sec\n",
            "epoch 20, iter 7620, avg. loss 27.01, avg. ppl 3.66 cum. examples 6716, speed 1303.97 words/sec, time elapsed 2030.63 sec\n",
            "epoch 20, iter 7630, avg. loss 23.43, avg. ppl 2.81 cum. examples 6876, speed 1245.36 words/sec, time elapsed 2033.54 sec\n",
            "epoch 20, iter 7640, avg. loss 21.69, avg. ppl 2.69 cum. examples 7036, speed 1356.87 words/sec, time elapsed 2036.13 sec\n",
            "epoch 20, iter 7650, avg. loss 19.92, avg. ppl 2.58 cum. examples 7196, speed 1372.22 words/sec, time elapsed 2038.57 sec\n",
            "epoch 20, iter 7660, avg. loss 22.16, avg. ppl 2.78 cum. examples 7356, speed 1389.38 words/sec, time elapsed 2041.07 sec\n",
            "epoch 20, iter 7670, avg. loss 31.80, avg. ppl 3.60 cum. examples 7516, speed 1218.52 words/sec, time elapsed 2044.33 sec\n",
            "epoch 20, iter 7680, avg. loss 28.63, avg. ppl 3.36 cum. examples 7676, speed 1325.53 words/sec, time elapsed 2047.18 sec\n",
            "epoch 20, iter 7690, avg. loss 20.09, avg. ppl 2.60 cum. examples 7836, speed 1426.84 words/sec, time elapsed 2049.55 sec\n",
            "epoch 20, iter 7700, avg. loss 23.54, avg. ppl 2.80 cum. examples 7996, speed 1474.44 words/sec, time elapsed 2052.03 sec\n",
            "epoch 20, iter 7710, avg. loss 27.49, avg. ppl 3.22 cum. examples 8156, speed 1248.74 words/sec, time elapsed 2055.04 sec\n",
            "epoch 20, iter 7720, avg. loss 18.65, avg. ppl 2.51 cum. examples 8316, speed 1246.02 words/sec, time elapsed 2057.64 sec\n",
            "epoch 20, iter 7730, avg. loss 23.96, avg. ppl 2.98 cum. examples 8476, speed 1142.25 words/sec, time elapsed 2060.72 sec\n",
            "epoch 20, iter 7740, avg. loss 24.18, avg. ppl 2.83 cum. examples 8636, speed 1459.41 words/sec, time elapsed 2063.27 sec\n",
            "epoch 20, iter 7750, avg. loss 19.55, avg. ppl 2.57 cum. examples 8796, speed 1563.55 words/sec, time elapsed 2065.38 sec\n",
            "epoch 20, iter 7760, avg. loss 27.94, avg. ppl 3.12 cum. examples 8956, speed 1435.39 words/sec, time elapsed 2068.12 sec\n",
            "epoch 20, iter 7770, avg. loss 23.60, avg. ppl 2.73 cum. examples 9116, speed 1539.41 words/sec, time elapsed 2070.56 sec\n",
            "epoch 20, iter 7780, avg. loss 20.11, avg. ppl 2.59 cum. examples 9276, speed 1283.34 words/sec, time elapsed 2073.19 sec\n",
            "epoch 20, iter 7790, avg. loss 20.46, avg. ppl 2.66 cum. examples 9436, speed 1484.92 words/sec, time elapsed 2075.44 sec\n",
            "epoch 20, iter 7800, avg. loss 21.36, avg. ppl 2.74 cum. examples 9596, speed 1363.42 words/sec, time elapsed 2077.93 sec\n",
            "epoch 20, iter 7810, avg. loss 18.73, avg. ppl 2.47 cum. examples 9756, speed 1592.72 words/sec, time elapsed 2080.01 sec\n",
            "epoch 20, iter 7820, avg. loss 22.93, avg. ppl 2.83 cum. examples 9916, speed 1477.71 words/sec, time elapsed 2082.40 sec\n",
            "epoch 20, iter 7830, avg. loss 24.88, avg. ppl 3.17 cum. examples 10076, speed 1377.59 words/sec, time elapsed 2084.91 sec\n",
            "epoch 20, iter 7840, avg. loss 26.52, avg. ppl 3.27 cum. examples 10236, speed 1243.78 words/sec, time elapsed 2087.79 sec\n",
            "epoch 20, iter 7850, avg. loss 21.86, avg. ppl 2.75 cum. examples 10396, speed 1482.05 words/sec, time elapsed 2090.12 sec\n",
            "epoch 20, iter 7860, avg. loss 20.54, avg. ppl 2.59 cum. examples 10556, speed 1660.06 words/sec, time elapsed 2092.20 sec\n",
            "epoch 20, iter 7870, avg. loss 27.34, avg. ppl 3.25 cum. examples 10716, speed 1473.84 words/sec, time elapsed 2094.72 sec\n",
            "epoch 20, iter 7880, avg. loss 30.41, avg. ppl 3.54 cum. examples 10876, speed 1340.18 words/sec, time elapsed 2097.59 sec\n",
            "epoch 20, iter 7890, avg. loss 22.13, avg. ppl 2.84 cum. examples 11036, speed 1361.19 words/sec, time elapsed 2100.08 sec\n",
            "epoch 20, iter 7900, avg. loss 23.91, avg. ppl 2.92 cum. examples 11196, speed 1410.38 words/sec, time elapsed 2102.61 sec\n",
            "epoch 20, iter 7910, avg. loss 21.01, avg. ppl 2.70 cum. examples 11356, speed 1619.25 words/sec, time elapsed 2104.70 sec\n",
            "epoch 20, iter 7920, avg. loss 27.61, avg. ppl 3.27 cum. examples 11516, speed 1326.27 words/sec, time elapsed 2107.52 sec\n",
            "epoch 20, iter 7930, avg. loss 21.74, avg. ppl 2.91 cum. examples 11676, speed 1148.77 words/sec, time elapsed 2110.36 sec\n",
            "epoch 20, iter 7940, avg. loss 22.77, avg. ppl 2.81 cum. examples 11836, speed 1429.16 words/sec, time elapsed 2112.82 sec\n",
            "epoch 20, iter 7950, avg. loss 26.57, avg. ppl 3.30 cum. examples 11996, speed 1118.99 words/sec, time elapsed 2116.01 sec\n",
            "epoch 20, iter 7960, avg. loss 24.31, avg. ppl 3.15 cum. examples 12156, speed 1168.61 words/sec, time elapsed 2118.91 sec\n",
            "epoch 20, iter 7970, avg. loss 22.86, avg. ppl 2.98 cum. examples 12316, speed 1390.12 words/sec, time elapsed 2121.33 sec\n",
            "epoch 20, iter 7980, avg. loss 21.46, avg. ppl 2.76 cum. examples 12476, speed 1522.10 words/sec, time elapsed 2123.54 sec\n",
            "epoch 20, iter 7990, avg. loss 30.18, avg. ppl 3.52 cum. examples 12636, speed 1348.41 words/sec, time elapsed 2126.39 sec\n",
            "epoch 20, iter 8000, avg. loss 26.69, avg. ppl 3.30 cum. examples 12796, speed 1173.02 words/sec, time elapsed 2129.44 sec\n",
            "epoch 20, iter 8000, cum. loss 26.66, cum. ppl 3.34 cum. examples 12796\n",
            "begin validation ...\n",
            "validation: iter 8000, dev. ppl 202.952946\n",
            "hit patience 4\n",
            "epoch 20, iter 8010, avg. loss 22.50, avg. ppl 2.87 cum. examples 160, speed 447.94 words/sec, time elapsed 2137.07 sec\n",
            "epoch 20, iter 8020, avg. loss 30.42, avg. ppl 3.68 cum. examples 318, speed 1413.86 words/sec, time elapsed 2139.68 sec\n",
            "epoch 21, iter 8030, avg. loss 17.70, avg. ppl 2.17 cum. examples 478, speed 1711.33 words/sec, time elapsed 2141.82 sec\n",
            "epoch 21, iter 8040, avg. loss 13.28, avg. ppl 1.94 cum. examples 638, speed 1487.85 words/sec, time elapsed 2143.97 sec\n",
            "epoch 21, iter 8050, avg. loss 18.21, avg. ppl 2.31 cum. examples 798, speed 1542.71 words/sec, time elapsed 2146.22 sec\n",
            "epoch 21, iter 8060, avg. loss 16.51, avg. ppl 2.19 cum. examples 958, speed 1419.66 words/sec, time elapsed 2148.59 sec\n",
            "epoch 21, iter 8070, avg. loss 22.28, avg. ppl 2.64 cum. examples 1118, speed 1268.78 words/sec, time elapsed 2151.49 sec\n",
            "epoch 21, iter 8080, avg. loss 23.32, avg. ppl 2.70 cum. examples 1278, speed 1339.84 words/sec, time elapsed 2154.29 sec\n",
            "epoch 21, iter 8090, avg. loss 21.59, avg. ppl 2.60 cum. examples 1438, speed 1207.15 words/sec, time elapsed 2157.29 sec\n",
            "epoch 21, iter 8100, avg. loss 21.98, avg. ppl 2.58 cum. examples 1598, speed 1307.88 words/sec, time elapsed 2160.12 sec\n",
            "epoch 21, iter 8110, avg. loss 19.62, avg. ppl 2.41 cum. examples 1758, speed 1401.87 words/sec, time elapsed 2162.67 sec\n",
            "epoch 21, iter 8120, avg. loss 15.43, avg. ppl 2.07 cum. examples 1918, speed 1536.14 words/sec, time elapsed 2164.88 sec\n",
            "epoch 21, iter 8130, avg. loss 19.69, avg. ppl 2.37 cum. examples 2078, speed 1480.93 words/sec, time elapsed 2167.34 sec\n",
            "epoch 21, iter 8140, avg. loss 20.04, avg. ppl 2.46 cum. examples 2238, speed 1371.86 words/sec, time elapsed 2169.94 sec\n",
            "epoch 21, iter 8150, avg. loss 21.39, avg. ppl 2.65 cum. examples 2398, speed 1395.61 words/sec, time elapsed 2172.46 sec\n",
            "epoch 21, iter 8160, avg. loss 15.09, avg. ppl 2.08 cum. examples 2558, speed 1392.20 words/sec, time elapsed 2174.82 sec\n",
            "epoch 21, iter 8170, avg. loss 18.58, avg. ppl 2.38 cum. examples 2718, speed 1404.28 words/sec, time elapsed 2177.27 sec\n",
            "epoch 21, iter 8180, avg. loss 17.11, avg. ppl 2.21 cum. examples 2878, speed 1256.35 words/sec, time elapsed 2180.02 sec\n",
            "epoch 21, iter 8190, avg. loss 15.41, avg. ppl 2.08 cum. examples 3038, speed 1427.10 words/sec, time elapsed 2182.37 sec\n",
            "epoch 21, iter 8200, avg. loss 23.99, avg. ppl 2.71 cum. examples 3198, speed 1310.11 words/sec, time elapsed 2185.31 sec\n",
            "epoch 21, iter 8210, avg. loss 19.97, avg. ppl 2.40 cum. examples 3358, speed 1317.81 words/sec, time elapsed 2188.08 sec\n",
            "epoch 21, iter 8220, avg. loss 15.34, avg. ppl 2.10 cum. examples 3518, speed 1299.36 words/sec, time elapsed 2190.63 sec\n",
            "epoch 21, iter 8230, avg. loss 16.40, avg. ppl 2.20 cum. examples 3678, speed 1225.82 words/sec, time elapsed 2193.34 sec\n",
            "epoch 21, iter 8240, avg. loss 20.67, avg. ppl 2.50 cum. examples 3838, speed 1356.55 words/sec, time elapsed 2196.00 sec\n",
            "epoch 21, iter 8250, avg. loss 15.31, avg. ppl 2.15 cum. examples 3998, speed 1303.52 words/sec, time elapsed 2198.46 sec\n",
            "epoch 21, iter 8260, avg. loss 21.19, avg. ppl 2.60 cum. examples 4158, speed 1292.22 words/sec, time elapsed 2201.21 sec\n",
            "epoch 21, iter 8270, avg. loss 14.41, avg. ppl 2.04 cum. examples 4318, speed 1446.80 words/sec, time elapsed 2203.44 sec\n",
            "epoch 21, iter 8280, avg. loss 22.45, avg. ppl 2.62 cum. examples 4478, speed 1511.93 words/sec, time elapsed 2205.91 sec\n",
            "epoch 21, iter 8290, avg. loss 24.25, avg. ppl 2.85 cum. examples 4638, speed 1398.67 words/sec, time elapsed 2208.56 sec\n",
            "epoch 21, iter 8300, avg. loss 23.99, avg. ppl 2.73 cum. examples 4798, speed 1304.12 words/sec, time elapsed 2211.49 sec\n",
            "epoch 21, iter 8310, avg. loss 22.92, avg. ppl 2.79 cum. examples 4958, speed 1104.41 words/sec, time elapsed 2214.73 sec\n",
            "epoch 21, iter 8320, avg. loss 21.77, avg. ppl 2.59 cum. examples 5118, speed 1252.60 words/sec, time elapsed 2217.65 sec\n",
            "epoch 21, iter 8330, avg. loss 18.00, avg. ppl 2.30 cum. examples 5278, speed 1671.48 words/sec, time elapsed 2219.72 sec\n",
            "epoch 21, iter 8340, avg. loss 18.91, avg. ppl 2.41 cum. examples 5438, speed 1430.72 words/sec, time elapsed 2222.13 sec\n",
            "epoch 21, iter 8350, avg. loss 21.01, avg. ppl 2.49 cum. examples 5598, speed 1529.43 words/sec, time elapsed 2224.54 sec\n",
            "epoch 21, iter 8360, avg. loss 23.72, avg. ppl 2.76 cum. examples 5758, speed 1322.57 words/sec, time elapsed 2227.37 sec\n",
            "epoch 21, iter 8370, avg. loss 23.45, avg. ppl 2.69 cum. examples 5918, speed 1417.97 words/sec, time elapsed 2230.04 sec\n",
            "epoch 21, iter 8380, avg. loss 19.53, avg. ppl 2.41 cum. examples 6078, speed 1509.01 words/sec, time elapsed 2232.39 sec\n",
            "epoch 21, iter 8390, avg. loss 23.74, avg. ppl 2.80 cum. examples 6238, speed 1232.40 words/sec, time elapsed 2235.38 sec\n",
            "epoch 21, iter 8400, avg. loss 18.62, avg. ppl 2.49 cum. examples 6398, speed 1243.69 words/sec, time elapsed 2238.01 sec\n",
            "epoch 21, iter 8410, avg. loss 19.27, avg. ppl 2.42 cum. examples 6558, speed 1461.80 words/sec, time elapsed 2240.40 sec\n",
            "epoch 21, iter 8420, avg. loss 20.09, avg. ppl 2.52 cum. examples 6718, speed 1276.78 words/sec, time elapsed 2243.12 sec\n",
            "epoch 22, iter 8430, avg. loss 13.98, avg. ppl 1.85 cum. examples 6876, speed 1525.04 words/sec, time elapsed 2245.48 sec\n",
            "epoch 22, iter 8440, avg. loss 15.36, avg. ppl 2.01 cum. examples 7036, speed 1404.36 words/sec, time elapsed 2247.98 sec\n",
            "epoch 22, iter 8450, avg. loss 19.38, avg. ppl 2.22 cum. examples 7196, speed 1284.59 words/sec, time elapsed 2251.01 sec\n",
            "epoch 22, iter 8460, avg. loss 16.01, avg. ppl 2.07 cum. examples 7356, speed 1351.66 words/sec, time elapsed 2253.62 sec\n",
            "epoch 22, iter 8470, avg. loss 13.03, avg. ppl 1.83 cum. examples 7516, speed 1466.37 words/sec, time elapsed 2255.97 sec\n",
            "epoch 22, iter 8480, avg. loss 17.07, avg. ppl 2.09 cum. examples 7676, speed 1349.97 words/sec, time elapsed 2258.72 sec\n",
            "epoch 22, iter 8490, avg. loss 14.96, avg. ppl 1.95 cum. examples 7836, speed 1341.65 words/sec, time elapsed 2261.39 sec\n",
            "epoch 22, iter 8500, avg. loss 16.47, avg. ppl 2.07 cum. examples 7996, speed 1421.88 words/sec, time elapsed 2263.94 sec\n",
            "epoch 22, iter 8510, avg. loss 14.30, avg. ppl 1.90 cum. examples 8156, speed 1537.16 words/sec, time elapsed 2266.25 sec\n",
            "epoch 22, iter 8520, avg. loss 15.68, avg. ppl 2.03 cum. examples 8316, speed 1213.08 words/sec, time elapsed 2269.17 sec\n",
            "epoch 22, iter 8530, avg. loss 12.80, avg. ppl 1.83 cum. examples 8476, speed 1359.56 words/sec, time elapsed 2271.66 sec\n",
            "epoch 22, iter 8540, avg. loss 16.48, avg. ppl 2.13 cum. examples 8636, speed 1173.42 words/sec, time elapsed 2274.63 sec\n",
            "epoch 22, iter 8550, avg. loss 14.42, avg. ppl 1.98 cum. examples 8796, speed 1249.66 words/sec, time elapsed 2277.34 sec\n",
            "epoch 22, iter 8560, avg. loss 14.17, avg. ppl 1.94 cum. examples 8956, speed 1402.42 words/sec, time elapsed 2279.77 sec\n",
            "epoch 22, iter 8570, avg. loss 18.21, avg. ppl 2.13 cum. examples 9116, speed 1374.29 words/sec, time elapsed 2282.58 sec\n",
            "epoch 22, iter 8580, avg. loss 14.90, avg. ppl 1.97 cum. examples 9276, speed 1443.01 words/sec, time elapsed 2285.02 sec\n",
            "epoch 22, iter 8590, avg. loss 21.89, avg. ppl 2.64 cum. examples 9436, speed 1135.76 words/sec, time elapsed 2288.19 sec\n",
            "epoch 22, iter 8600, avg. loss 15.28, avg. ppl 1.95 cum. examples 9596, speed 1363.31 words/sec, time elapsed 2290.87 sec\n",
            "epoch 22, iter 8610, avg. loss 13.98, avg. ppl 1.91 cum. examples 9756, speed 1440.22 words/sec, time elapsed 2293.28 sec\n",
            "epoch 22, iter 8620, avg. loss 13.90, avg. ppl 1.89 cum. examples 9916, speed 1550.39 words/sec, time elapsed 2295.54 sec\n",
            "epoch 22, iter 8630, avg. loss 12.12, avg. ppl 1.81 cum. examples 10076, speed 1537.77 words/sec, time elapsed 2297.67 sec\n",
            "epoch 22, iter 8640, avg. loss 13.91, avg. ppl 1.87 cum. examples 10236, speed 1679.29 words/sec, time elapsed 2299.78 sec\n",
            "epoch 22, iter 8650, avg. loss 14.82, avg. ppl 1.98 cum. examples 10396, speed 1480.90 words/sec, time elapsed 2302.13 sec\n",
            "epoch 22, iter 8660, avg. loss 12.20, avg. ppl 1.82 cum. examples 10556, speed 1328.12 words/sec, time elapsed 2304.58 sec\n",
            "epoch 22, iter 8670, avg. loss 17.53, avg. ppl 2.16 cum. examples 10716, speed 1403.61 words/sec, time elapsed 2307.17 sec\n",
            "epoch 22, iter 8680, avg. loss 13.61, avg. ppl 1.93 cum. examples 10876, speed 1343.28 words/sec, time elapsed 2309.63 sec\n",
            "epoch 22, iter 8690, avg. loss 17.83, avg. ppl 2.24 cum. examples 11036, speed 1230.16 words/sec, time elapsed 2312.50 sec\n",
            "epoch 22, iter 8700, avg. loss 15.76, avg. ppl 2.04 cum. examples 11196, speed 1354.85 words/sec, time elapsed 2315.11 sec\n",
            "epoch 22, iter 8710, avg. loss 18.41, avg. ppl 2.25 cum. examples 11356, speed 1079.30 words/sec, time elapsed 2318.48 sec\n",
            "epoch 22, iter 8720, avg. loss 16.05, avg. ppl 2.06 cum. examples 11516, speed 1226.36 words/sec, time elapsed 2321.38 sec\n",
            "epoch 22, iter 8730, avg. loss 19.09, avg. ppl 2.29 cum. examples 11676, speed 1318.58 words/sec, time elapsed 2324.17 sec\n",
            "epoch 22, iter 8740, avg. loss 13.44, avg. ppl 1.87 cum. examples 11836, speed 1466.54 words/sec, time elapsed 2326.51 sec\n",
            "epoch 22, iter 8750, avg. loss 22.80, avg. ppl 2.51 cum. examples 11996, speed 1235.93 words/sec, time elapsed 2329.72 sec\n",
            "epoch 22, iter 8760, avg. loss 10.36, avg. ppl 1.73 cum. examples 12156, speed 1388.73 words/sec, time elapsed 2331.89 sec\n",
            "epoch 22, iter 8770, avg. loss 17.19, avg. ppl 2.23 cum. examples 12316, speed 1321.90 words/sec, time elapsed 2334.49 sec\n",
            "epoch 22, iter 8780, avg. loss 17.55, avg. ppl 2.19 cum. examples 12476, speed 1350.99 words/sec, time elapsed 2337.14 sec\n",
            "epoch 22, iter 8790, avg. loss 21.74, avg. ppl 2.55 cum. examples 12636, speed 1122.76 words/sec, time elapsed 2340.44 sec\n",
            "epoch 22, iter 8800, avg. loss 16.88, avg. ppl 2.15 cum. examples 12796, speed 1408.01 words/sec, time elapsed 2342.95 sec\n",
            "epoch 22, iter 8800, cum. loss 18.03, cum. ppl 2.26 cum. examples 12796\n",
            "begin validation ...\n",
            "validation: iter 8800, dev. ppl 290.156657\n",
            "hit patience 5\n",
            "hit #2 trial\n",
            "load previously best model and decay learning rate to 0.000250\n",
            "restore parameters of the optimizers\n",
            "epoch 22, iter 8810, avg. loss 80.17, avg. ppl 38.53 cum. examples 160, speed 425.27 words/sec, time elapsed 2351.21 sec\n",
            "epoch 22, iter 8820, avg. loss 78.73, avg. ppl 37.69 cum. examples 320, speed 1376.90 words/sec, time elapsed 2353.73 sec\n",
            "epoch 23, iter 8830, avg. loss 81.95, avg. ppl 40.69 cum. examples 478, speed 1160.54 words/sec, time elapsed 2356.74 sec\n",
            "epoch 23, iter 8840, avg. loss 76.87, avg. ppl 37.16 cum. examples 638, speed 1460.55 words/sec, time elapsed 2359.07 sec\n",
            "epoch 23, iter 8850, avg. loss 86.74, avg. ppl 39.17 cum. examples 798, speed 1293.55 words/sec, time elapsed 2362.00 sec\n",
            "epoch 23, iter 8860, avg. loss 71.10, avg. ppl 35.37 cum. examples 958, speed 1450.15 words/sec, time elapsed 2364.20 sec\n",
            "epoch 23, iter 8870, avg. loss 74.00, avg. ppl 33.49 cum. examples 1118, speed 1421.93 words/sec, time elapsed 2366.57 sec\n",
            "epoch 23, iter 8880, avg. loss 73.55, avg. ppl 32.32 cum. examples 1278, speed 1403.84 words/sec, time elapsed 2368.98 sec\n",
            "epoch 23, iter 8890, avg. loss 76.02, avg. ppl 32.47 cum. examples 1438, speed 1425.08 words/sec, time elapsed 2371.43 sec\n",
            "epoch 23, iter 8900, avg. loss 81.01, avg. ppl 36.40 cum. examples 1598, speed 1217.92 words/sec, time elapsed 2374.39 sec\n",
            "epoch 23, iter 8910, avg. loss 76.42, avg. ppl 33.97 cum. examples 1758, speed 1483.05 words/sec, time elapsed 2376.73 sec\n",
            "epoch 23, iter 8920, avg. loss 78.80, avg. ppl 33.74 cum. examples 1918, speed 1448.35 words/sec, time elapsed 2379.21 sec\n",
            "epoch 23, iter 8930, avg. loss 77.40, avg. ppl 32.96 cum. examples 2078, speed 1438.45 words/sec, time elapsed 2381.67 sec\n",
            "epoch 23, iter 8940, avg. loss 73.69, avg. ppl 31.14 cum. examples 2238, speed 1075.07 words/sec, time elapsed 2384.86 sec\n",
            "epoch 23, iter 8950, avg. loss 82.62, avg. ppl 37.07 cum. examples 2398, speed 1304.18 words/sec, time elapsed 2387.66 sec\n",
            "epoch 23, iter 8960, avg. loss 81.86, avg. ppl 35.20 cum. examples 2558, speed 1421.69 words/sec, time elapsed 2390.25 sec\n",
            "epoch 23, iter 8970, avg. loss 76.01, avg. ppl 33.95 cum. examples 2718, speed 1360.39 words/sec, time elapsed 2392.79 sec\n",
            "epoch 23, iter 8980, avg. loss 76.54, avg. ppl 31.62 cum. examples 2878, speed 1421.14 words/sec, time elapsed 2395.28 sec\n",
            "epoch 23, iter 8990, avg. loss 75.78, avg. ppl 34.15 cum. examples 3038, speed 1755.25 words/sec, time elapsed 2397.24 sec\n",
            "epoch 23, iter 9000, avg. loss 73.52, avg. ppl 33.15 cum. examples 3198, speed 1365.42 words/sec, time elapsed 2399.70 sec\n",
            "epoch 23, iter 9010, avg. loss 76.49, avg. ppl 32.68 cum. examples 3358, speed 1410.47 words/sec, time elapsed 2402.19 sec\n",
            "epoch 23, iter 9020, avg. loss 75.54, avg. ppl 31.17 cum. examples 3518, speed 1343.55 words/sec, time elapsed 2404.80 sec\n",
            "epoch 23, iter 9030, avg. loss 85.28, avg. ppl 39.24 cum. examples 3678, speed 1311.22 words/sec, time elapsed 2407.64 sec\n",
            "epoch 23, iter 9040, avg. loss 74.03, avg. ppl 33.72 cum. examples 3838, speed 1287.05 words/sec, time elapsed 2410.26 sec\n",
            "epoch 23, iter 9050, avg. loss 82.96, avg. ppl 33.69 cum. examples 3998, speed 1469.44 words/sec, time elapsed 2412.82 sec\n",
            "epoch 23, iter 9060, avg. loss 84.48, avg. ppl 36.04 cum. examples 4158, speed 1573.81 words/sec, time elapsed 2415.22 sec\n",
            "epoch 23, iter 9070, avg. loss 78.70, avg. ppl 34.07 cum. examples 4318, speed 1189.39 words/sec, time elapsed 2418.22 sec\n",
            "epoch 23, iter 9080, avg. loss 82.56, avg. ppl 34.39 cum. examples 4478, speed 1246.92 words/sec, time elapsed 2421.22 sec\n",
            "epoch 23, iter 9090, avg. loss 83.16, avg. ppl 36.35 cum. examples 4638, speed 1289.32 words/sec, time elapsed 2424.09 sec\n",
            "epoch 23, iter 9100, avg. loss 79.56, avg. ppl 36.94 cum. examples 4798, speed 1448.78 words/sec, time elapsed 2426.52 sec\n",
            "epoch 23, iter 9110, avg. loss 81.49, avg. ppl 34.15 cum. examples 4958, speed 1218.31 words/sec, time elapsed 2429.55 sec\n",
            "epoch 23, iter 9120, avg. loss 73.98, avg. ppl 28.86 cum. examples 5118, speed 1326.82 words/sec, time elapsed 2432.21 sec\n",
            "epoch 23, iter 9130, avg. loss 73.78, avg. ppl 34.93 cum. examples 5278, speed 1408.48 words/sec, time elapsed 2434.57 sec\n",
            "epoch 23, iter 9140, avg. loss 76.58, avg. ppl 33.31 cum. examples 5438, speed 1282.93 words/sec, time elapsed 2437.29 sec\n",
            "epoch 23, iter 9150, avg. loss 74.84, avg. ppl 34.63 cum. examples 5598, speed 1363.60 words/sec, time elapsed 2439.77 sec\n",
            "epoch 23, iter 9160, avg. loss 84.25, avg. ppl 36.22 cum. examples 5758, speed 1537.80 words/sec, time elapsed 2442.21 sec\n",
            "epoch 23, iter 9170, avg. loss 76.74, avg. ppl 34.87 cum. examples 5918, speed 1361.19 words/sec, time elapsed 2444.75 sec\n",
            "epoch 23, iter 9180, avg. loss 75.70, avg. ppl 31.40 cum. examples 6078, speed 1430.56 words/sec, time elapsed 2447.21 sec\n",
            "epoch 23, iter 9190, avg. loss 74.57, avg. ppl 33.94 cum. examples 6238, speed 1344.20 words/sec, time elapsed 2449.72 sec\n",
            "epoch 23, iter 9200, avg. loss 73.71, avg. ppl 33.00 cum. examples 6398, speed 1371.65 words/sec, time elapsed 2452.18 sec\n",
            "epoch 23, iter 9210, avg. loss 90.24, avg. ppl 39.52 cum. examples 6558, speed 1173.70 words/sec, time elapsed 2455.53 sec\n",
            "epoch 23, iter 9220, avg. loss 80.67, avg. ppl 35.39 cum. examples 6718, speed 1325.25 words/sec, time elapsed 2458.26 sec\n",
            "epoch 24, iter 9230, avg. loss 69.43, avg. ppl 24.86 cum. examples 6876, speed 1244.66 words/sec, time elapsed 2461.00 sec\n",
            "epoch 24, iter 9240, avg. loss 69.96, avg. ppl 24.15 cum. examples 7036, speed 1343.89 words/sec, time elapsed 2463.62 sec\n",
            "epoch 24, iter 9250, avg. loss 76.24, avg. ppl 26.88 cum. examples 7196, speed 1249.32 words/sec, time elapsed 2466.58 sec\n",
            "epoch 24, iter 9260, avg. loss 65.31, avg. ppl 21.74 cum. examples 7356, speed 1609.73 words/sec, time elapsed 2468.69 sec\n",
            "epoch 24, iter 9270, avg. loss 69.96, avg. ppl 23.90 cum. examples 7516, speed 1329.49 words/sec, time elapsed 2471.35 sec\n",
            "epoch 24, iter 9280, avg. loss 73.98, avg. ppl 26.23 cum. examples 7676, speed 1355.79 words/sec, time elapsed 2474.02 sec\n",
            "epoch 24, iter 9290, avg. loss 67.58, avg. ppl 23.92 cum. examples 7836, speed 1295.42 words/sec, time elapsed 2476.65 sec\n",
            "epoch 24, iter 9300, avg. loss 74.53, avg. ppl 26.00 cum. examples 7996, speed 1500.94 words/sec, time elapsed 2479.09 sec\n",
            "epoch 24, iter 9310, avg. loss 65.38, avg. ppl 21.86 cum. examples 8156, speed 1302.85 words/sec, time elapsed 2481.69 sec\n",
            "epoch 24, iter 9320, avg. loss 69.17, avg. ppl 22.89 cum. examples 8316, speed 1452.42 words/sec, time elapsed 2484.12 sec\n",
            "epoch 24, iter 9330, avg. loss 65.03, avg. ppl 20.76 cum. examples 8476, speed 1465.85 words/sec, time elapsed 2486.46 sec\n",
            "epoch 24, iter 9340, avg. loss 74.07, avg. ppl 26.44 cum. examples 8636, speed 1197.21 words/sec, time elapsed 2489.49 sec\n",
            "epoch 24, iter 9350, avg. loss 69.26, avg. ppl 24.50 cum. examples 8796, speed 1300.66 words/sec, time elapsed 2492.15 sec\n",
            "epoch 24, iter 9360, avg. loss 69.71, avg. ppl 25.14 cum. examples 8956, speed 1219.27 words/sec, time elapsed 2494.99 sec\n",
            "epoch 24, iter 9370, avg. loss 67.05, avg. ppl 24.06 cum. examples 9116, speed 1425.67 words/sec, time elapsed 2497.35 sec\n",
            "epoch 24, iter 9380, avg. loss 59.95, avg. ppl 20.61 cum. examples 9276, speed 1605.67 words/sec, time elapsed 2499.33 sec\n",
            "epoch 24, iter 9390, avg. loss 75.31, avg. ppl 25.16 cum. examples 9436, speed 1496.10 words/sec, time elapsed 2501.82 sec\n",
            "epoch 24, iter 9400, avg. loss 74.21, avg. ppl 24.84 cum. examples 9596, speed 1269.77 words/sec, time elapsed 2504.73 sec\n",
            "epoch 24, iter 9410, avg. loss 72.24, avg. ppl 23.87 cum. examples 9756, speed 1428.39 words/sec, time elapsed 2507.29 sec\n",
            "epoch 24, iter 9420, avg. loss 77.59, avg. ppl 26.39 cum. examples 9916, speed 1287.43 words/sec, time elapsed 2510.23 sec\n",
            "epoch 24, iter 9430, avg. loss 74.80, avg. ppl 25.44 cum. examples 10076, speed 1341.61 words/sec, time elapsed 2512.99 sec\n",
            "epoch 24, iter 9440, avg. loss 67.31, avg. ppl 22.91 cum. examples 10236, speed 1275.18 words/sec, time elapsed 2515.68 sec\n",
            "epoch 24, iter 9450, avg. loss 72.92, avg. ppl 26.38 cum. examples 10396, speed 1172.46 words/sec, time elapsed 2518.73 sec\n",
            "epoch 24, iter 9460, avg. loss 77.09, avg. ppl 28.17 cum. examples 10556, speed 1190.09 words/sec, time elapsed 2521.83 sec\n",
            "epoch 24, iter 9470, avg. loss 76.07, avg. ppl 27.83 cum. examples 10716, speed 1378.83 words/sec, time elapsed 2524.48 sec\n",
            "epoch 24, iter 9480, avg. loss 60.22, avg. ppl 22.02 cum. examples 10876, speed 1520.13 words/sec, time elapsed 2526.53 sec\n",
            "epoch 24, iter 9490, avg. loss 73.57, avg. ppl 24.67 cum. examples 11036, speed 1401.42 words/sec, time elapsed 2529.15 sec\n",
            "epoch 24, iter 9500, avg. loss 81.26, avg. ppl 28.39 cum. examples 11196, speed 1651.39 words/sec, time elapsed 2531.51 sec\n",
            "epoch 24, iter 9510, avg. loss 79.40, avg. ppl 29.15 cum. examples 11356, speed 1281.28 words/sec, time elapsed 2534.45 sec\n",
            "epoch 24, iter 9520, avg. loss 68.72, avg. ppl 24.99 cum. examples 11516, speed 1422.64 words/sec, time elapsed 2536.85 sec\n",
            "epoch 24, iter 9530, avg. loss 72.38, avg. ppl 25.89 cum. examples 11676, speed 1529.57 words/sec, time elapsed 2539.18 sec\n",
            "epoch 24, iter 9540, avg. loss 64.70, avg. ppl 23.08 cum. examples 11836, speed 1328.32 words/sec, time elapsed 2541.66 sec\n",
            "epoch 24, iter 9550, avg. loss 67.18, avg. ppl 24.65 cum. examples 11996, speed 1459.30 words/sec, time elapsed 2543.96 sec\n",
            "epoch 24, iter 9560, avg. loss 69.79, avg. ppl 25.43 cum. examples 12156, speed 1511.69 words/sec, time elapsed 2546.24 sec\n",
            "epoch 24, iter 9570, avg. loss 62.94, avg. ppl 21.71 cum. examples 12316, speed 1500.20 words/sec, time elapsed 2548.42 sec\n",
            "epoch 24, iter 9580, avg. loss 73.27, avg. ppl 27.18 cum. examples 12476, speed 1480.43 words/sec, time elapsed 2550.82 sec\n",
            "epoch 24, iter 9590, avg. loss 68.15, avg. ppl 24.32 cum. examples 12636, speed 1453.27 words/sec, time elapsed 2553.17 sec\n",
            "epoch 24, iter 9600, avg. loss 64.24, avg. ppl 21.84 cum. examples 12796, speed 1411.17 words/sec, time elapsed 2555.53 sec\n",
            "epoch 24, iter 9600, cum. loss 74.65, cum. ppl 29.52 cum. examples 12796\n",
            "begin validation ...\n",
            "validation: iter 9600, dev. ppl 58.776069\n",
            "hit patience 1\n",
            "epoch 24, iter 9610, avg. loss 81.99, avg. ppl 28.79 cum. examples 160, speed 475.18 words/sec, time elapsed 2563.75 sec\n",
            "epoch 24, iter 9620, avg. loss 73.97, avg. ppl 26.05 cum. examples 320, speed 1300.96 words/sec, time elapsed 2566.54 sec\n",
            "epoch 25, iter 9630, avg. loss 74.97, avg. ppl 22.86 cum. examples 478, speed 1465.42 words/sec, time elapsed 2569.12 sec\n",
            "epoch 25, iter 9640, avg. loss 59.37, avg. ppl 16.30 cum. examples 638, speed 1572.81 words/sec, time elapsed 2571.28 sec\n",
            "epoch 25, iter 9650, avg. loss 60.31, avg. ppl 16.27 cum. examples 798, speed 1416.36 words/sec, time elapsed 2573.73 sec\n",
            "epoch 25, iter 9660, avg. loss 53.66, avg. ppl 15.02 cum. examples 958, speed 1493.14 words/sec, time elapsed 2575.85 sec\n",
            "epoch 25, iter 9670, avg. loss 65.83, avg. ppl 17.82 cum. examples 1118, speed 1349.62 words/sec, time elapsed 2578.56 sec\n",
            "epoch 25, iter 9680, avg. loss 56.55, avg. ppl 15.47 cum. examples 1278, speed 1399.65 words/sec, time elapsed 2580.92 sec\n",
            "epoch 25, iter 9690, avg. loss 70.39, avg. ppl 19.03 cum. examples 1438, speed 1373.90 words/sec, time elapsed 2583.70 sec\n",
            "epoch 25, iter 9700, avg. loss 60.05, avg. ppl 16.85 cum. examples 1598, speed 1368.95 words/sec, time elapsed 2586.19 sec\n",
            "epoch 25, iter 9710, avg. loss 71.48, avg. ppl 20.49 cum. examples 1758, speed 1245.46 words/sec, time elapsed 2589.23 sec\n",
            "epoch 25, iter 9720, avg. loss 64.97, avg. ppl 18.19 cum. examples 1918, speed 1236.08 words/sec, time elapsed 2592.13 sec\n",
            "epoch 25, iter 9730, avg. loss 59.09, avg. ppl 16.10 cum. examples 2078, speed 1421.73 words/sec, time elapsed 2594.52 sec\n",
            "epoch 25, iter 9740, avg. loss 65.24, avg. ppl 18.17 cum. examples 2238, speed 1464.18 words/sec, time elapsed 2596.98 sec\n",
            "epoch 25, iter 9750, avg. loss 59.20, avg. ppl 16.68 cum. examples 2398, speed 1254.47 words/sec, time elapsed 2599.66 sec\n",
            "epoch 25, iter 9760, avg. loss 64.95, avg. ppl 18.04 cum. examples 2558, speed 1344.26 words/sec, time elapsed 2602.34 sec\n",
            "epoch 25, iter 9770, avg. loss 72.94, avg. ppl 20.53 cum. examples 2718, speed 1387.85 words/sec, time elapsed 2605.12 sec\n",
            "epoch 25, iter 9780, avg. loss 61.12, avg. ppl 18.49 cum. examples 2878, speed 1459.03 words/sec, time elapsed 2607.42 sec\n",
            "epoch 25, iter 9790, avg. loss 61.06, avg. ppl 16.83 cum. examples 3038, speed 1653.43 words/sec, time elapsed 2609.51 sec\n",
            "epoch 25, iter 9800, avg. loss 72.50, avg. ppl 19.97 cum. examples 3198, speed 1257.88 words/sec, time elapsed 2612.59 sec\n",
            "epoch 25, iter 9810, avg. loss 65.89, avg. ppl 17.53 cum. examples 3358, speed 1465.35 words/sec, time elapsed 2615.10 sec\n",
            "epoch 25, iter 9820, avg. loss 65.54, avg. ppl 19.26 cum. examples 3518, speed 1254.79 words/sec, time elapsed 2617.93 sec\n",
            "epoch 25, iter 9830, avg. loss 61.85, avg. ppl 18.29 cum. examples 3678, speed 1332.44 words/sec, time elapsed 2620.48 sec\n",
            "epoch 25, iter 9840, avg. loss 66.04, avg. ppl 18.95 cum. examples 3838, speed 1173.20 words/sec, time elapsed 2623.54 sec\n",
            "epoch 25, iter 9850, avg. loss 57.05, avg. ppl 16.29 cum. examples 3998, speed 1324.83 words/sec, time elapsed 2626.01 sec\n",
            "epoch 25, iter 9860, avg. loss 62.02, avg. ppl 18.59 cum. examples 4158, speed 1174.91 words/sec, time elapsed 2628.90 sec\n",
            "epoch 25, iter 9870, avg. loss 60.85, avg. ppl 17.55 cum. examples 4318, speed 1375.76 words/sec, time elapsed 2631.37 sec\n",
            "epoch 25, iter 9880, avg. loss 67.27, avg. ppl 18.43 cum. examples 4478, speed 1253.58 words/sec, time elapsed 2634.32 sec\n",
            "epoch 25, iter 9890, avg. loss 67.14, avg. ppl 19.90 cum. examples 4638, speed 1355.78 words/sec, time elapsed 2636.97 sec\n",
            "epoch 25, iter 9900, avg. loss 65.89, avg. ppl 19.07 cum. examples 4798, speed 1366.31 words/sec, time elapsed 2639.59 sec\n",
            "epoch 25, iter 9910, avg. loss 62.91, avg. ppl 17.23 cum. examples 4958, speed 1416.85 words/sec, time elapsed 2642.08 sec\n",
            "epoch 25, iter 9920, avg. loss 58.17, avg. ppl 17.11 cum. examples 5118, speed 1376.20 words/sec, time elapsed 2644.46 sec\n",
            "epoch 25, iter 9930, avg. loss 69.00, avg. ppl 19.37 cum. examples 5278, speed 1297.86 words/sec, time elapsed 2647.33 sec\n",
            "epoch 25, iter 9940, avg. loss 64.55, avg. ppl 18.68 cum. examples 5438, speed 1471.43 words/sec, time elapsed 2649.73 sec\n",
            "epoch 25, iter 9950, avg. loss 62.22, avg. ppl 18.16 cum. examples 5598, speed 1576.51 words/sec, time elapsed 2651.91 sec\n",
            "epoch 25, iter 9960, avg. loss 62.21, avg. ppl 19.10 cum. examples 5758, speed 1446.08 words/sec, time elapsed 2654.24 sec\n",
            "epoch 25, iter 9970, avg. loss 65.93, avg. ppl 19.55 cum. examples 5918, speed 1345.86 words/sec, time elapsed 2656.88 sec\n",
            "epoch 25, iter 9980, avg. loss 64.67, avg. ppl 19.07 cum. examples 6078, speed 1337.83 words/sec, time elapsed 2659.50 sec\n",
            "epoch 25, iter 9990, avg. loss 60.60, avg. ppl 18.88 cum. examples 6238, speed 1251.96 words/sec, time elapsed 2662.14 sec\n",
            "epoch 25, iter 10000, avg. loss 67.11, avg. ppl 19.92 cum. examples 6398, speed 1418.04 words/sec, time elapsed 2664.67 sec\n",
            "epoch 25, iter 10010, avg. loss 66.99, avg. ppl 17.17 cum. examples 6558, speed 1317.35 words/sec, time elapsed 2667.53 sec\n",
            "epoch 25, iter 10020, avg. loss 72.38, avg. ppl 20.22 cum. examples 6718, speed 1290.35 words/sec, time elapsed 2670.52 sec\n",
            "epoch 26, iter 10030, avg. loss 68.22, avg. ppl 17.87 cum. examples 6876, speed 1346.03 words/sec, time elapsed 2673.29 sec\n",
            "epoch 26, iter 10040, avg. loss 57.55, avg. ppl 12.88 cum. examples 7036, speed 1328.05 words/sec, time elapsed 2676.01 sec\n",
            "epoch 26, iter 10050, avg. loss 62.82, avg. ppl 15.21 cum. examples 7196, speed 1213.31 words/sec, time elapsed 2679.05 sec\n",
            "epoch 26, iter 10060, avg. loss 58.30, avg. ppl 13.13 cum. examples 7356, speed 1366.59 words/sec, time elapsed 2681.70 sec\n",
            "epoch 26, iter 10070, avg. loss 49.95, avg. ppl 11.44 cum. examples 7516, speed 1293.50 words/sec, time elapsed 2684.24 sec\n",
            "epoch 26, iter 10080, avg. loss 56.92, avg. ppl 13.13 cum. examples 7676, speed 1444.98 words/sec, time elapsed 2686.69 sec\n",
            "epoch 26, iter 10090, avg. loss 48.77, avg. ppl 10.79 cum. examples 7836, speed 1201.42 words/sec, time elapsed 2689.42 sec\n",
            "epoch 26, iter 10100, avg. loss 72.12, avg. ppl 17.52 cum. examples 7996, speed 1439.44 words/sec, time elapsed 2692.22 sec\n",
            "epoch 26, iter 10110, avg. loss 55.36, avg. ppl 12.63 cum. examples 8156, speed 1656.65 words/sec, time elapsed 2694.32 sec\n",
            "epoch 26, iter 10120, avg. loss 53.68, avg. ppl 12.32 cum. examples 8316, speed 1397.48 words/sec, time elapsed 2696.77 sec\n",
            "epoch 26, iter 10130, avg. loss 57.76, avg. ppl 13.91 cum. examples 8476, speed 1523.92 words/sec, time elapsed 2699.08 sec\n",
            "epoch 26, iter 10140, avg. loss 64.18, avg. ppl 16.01 cum. examples 8636, speed 1248.03 words/sec, time elapsed 2702.04 sec\n",
            "epoch 26, iter 10150, avg. loss 61.74, avg. ppl 14.07 cum. examples 8796, speed 1267.61 words/sec, time elapsed 2704.99 sec\n",
            "epoch 26, iter 10160, avg. loss 58.22, avg. ppl 14.00 cum. examples 8956, speed 1388.29 words/sec, time elapsed 2707.53 sec\n",
            "epoch 26, iter 10170, avg. loss 52.43, avg. ppl 12.81 cum. examples 9116, speed 1489.20 words/sec, time elapsed 2709.74 sec\n",
            "epoch 26, iter 10180, avg. loss 49.18, avg. ppl 11.55 cum. examples 9276, speed 1481.81 words/sec, time elapsed 2711.91 sec\n",
            "epoch 26, iter 10190, avg. loss 59.48, avg. ppl 13.40 cum. examples 9436, speed 1344.10 words/sec, time elapsed 2714.64 sec\n",
            "epoch 26, iter 10200, avg. loss 56.68, avg. ppl 13.15 cum. examples 9596, speed 1460.79 words/sec, time elapsed 2717.05 sec\n",
            "epoch 26, iter 10210, avg. loss 59.88, avg. ppl 14.95 cum. examples 9756, speed 1402.75 words/sec, time elapsed 2719.57 sec\n",
            "epoch 26, iter 10220, avg. loss 63.87, avg. ppl 14.72 cum. examples 9916, speed 1332.49 words/sec, time elapsed 2722.43 sec\n",
            "epoch 26, iter 10230, avg. loss 49.28, avg. ppl 10.93 cum. examples 10076, speed 1402.05 words/sec, time elapsed 2724.78 sec\n",
            "epoch 26, iter 10240, avg. loss 60.44, avg. ppl 14.47 cum. examples 10236, speed 1341.61 words/sec, time elapsed 2727.48 sec\n",
            "epoch 26, iter 10250, avg. loss 56.64, avg. ppl 13.54 cum. examples 10396, speed 1350.54 words/sec, time elapsed 2730.05 sec\n",
            "epoch 26, iter 10260, avg. loss 62.62, avg. ppl 14.54 cum. examples 10556, speed 1324.52 words/sec, time elapsed 2732.88 sec\n",
            "epoch 26, iter 10270, avg. loss 62.31, avg. ppl 14.71 cum. examples 10716, speed 1451.23 words/sec, time elapsed 2735.43 sec\n",
            "epoch 26, iter 10280, avg. loss 52.66, avg. ppl 12.27 cum. examples 10876, speed 1442.44 words/sec, time elapsed 2737.76 sec\n",
            "epoch 26, iter 10290, avg. loss 62.63, avg. ppl 14.14 cum. examples 11036, speed 1354.01 words/sec, time elapsed 2740.56 sec\n",
            "epoch 26, iter 10300, avg. loss 60.10, avg. ppl 14.62 cum. examples 11196, speed 1338.02 words/sec, time elapsed 2743.24 sec\n",
            "epoch 26, iter 10310, avg. loss 56.81, avg. ppl 13.67 cum. examples 11356, speed 1265.77 words/sec, time elapsed 2745.98 sec\n",
            "epoch 26, iter 10320, avg. loss 67.06, avg. ppl 16.06 cum. examples 11516, speed 1290.42 words/sec, time elapsed 2748.98 sec\n",
            "epoch 26, iter 10330, avg. loss 62.95, avg. ppl 15.23 cum. examples 11676, speed 1307.99 words/sec, time elapsed 2751.80 sec\n",
            "epoch 26, iter 10340, avg. loss 53.51, avg. ppl 12.19 cum. examples 11836, speed 1252.79 words/sec, time elapsed 2754.54 sec\n",
            "epoch 26, iter 10350, avg. loss 56.22, avg. ppl 13.36 cum. examples 11996, speed 1275.66 words/sec, time elapsed 2757.26 sec\n",
            "epoch 26, iter 10360, avg. loss 50.63, avg. ppl 12.51 cum. examples 12156, speed 1245.11 words/sec, time elapsed 2759.83 sec\n",
            "epoch 26, iter 10370, avg. loss 63.55, avg. ppl 15.69 cum. examples 12316, speed 1228.27 words/sec, time elapsed 2762.84 sec\n",
            "epoch 26, iter 10380, avg. loss 52.57, avg. ppl 12.43 cum. examples 12476, speed 1538.14 words/sec, time elapsed 2765.01 sec\n",
            "epoch 26, iter 10390, avg. loss 59.97, avg. ppl 14.72 cum. examples 12636, speed 1224.35 words/sec, time elapsed 2767.92 sec\n",
            "epoch 26, iter 10400, avg. loss 54.92, avg. ppl 13.63 cum. examples 12796, speed 1547.62 words/sec, time elapsed 2770.10 sec\n",
            "epoch 26, iter 10400, cum. loss 61.72, cum. ppl 16.15 cum. examples 12796\n",
            "begin validation ...\n",
            "validation: iter 10400, dev. ppl 73.869561\n",
            "hit patience 2\n",
            "epoch 26, iter 10410, avg. loss 52.10, avg. ppl 12.56 cum. examples 160, speed 443.66 words/sec, time elapsed 2777.52 sec\n",
            "epoch 26, iter 10420, avg. loss 51.19, avg. ppl 12.21 cum. examples 320, speed 1477.92 words/sec, time elapsed 2779.74 sec\n",
            "epoch 27, iter 10430, avg. loss 52.10, avg. ppl 11.37 cum. examples 478, speed 1344.76 words/sec, time elapsed 2782.25 sec\n",
            "epoch 27, iter 10440, avg. loss 49.11, avg. ppl 9.14 cum. examples 638, speed 1678.48 words/sec, time elapsed 2784.37 sec\n",
            "epoch 27, iter 10450, avg. loss 49.07, avg. ppl 9.30 cum. examples 798, speed 1645.91 words/sec, time elapsed 2786.51 sec\n",
            "epoch 27, iter 10460, avg. loss 45.05, avg. ppl 9.41 cum. examples 958, speed 1387.24 words/sec, time elapsed 2788.83 sec\n",
            "epoch 27, iter 10470, avg. loss 54.38, avg. ppl 11.17 cum. examples 1118, speed 1162.04 words/sec, time elapsed 2791.93 sec\n",
            "epoch 27, iter 10480, avg. loss 46.26, avg. ppl 8.45 cum. examples 1278, speed 1490.84 words/sec, time elapsed 2794.26 sec\n",
            "epoch 27, iter 10490, avg. loss 45.92, avg. ppl 9.41 cum. examples 1438, speed 1425.19 words/sec, time elapsed 2796.56 sec\n",
            "epoch 27, iter 10500, avg. loss 53.52, avg. ppl 10.15 cum. examples 1598, speed 1507.22 words/sec, time elapsed 2799.01 sec\n",
            "epoch 27, iter 10510, avg. loss 42.14, avg. ppl 8.22 cum. examples 1758, speed 1464.69 words/sec, time elapsed 2801.19 sec\n",
            "epoch 27, iter 10520, avg. loss 56.20, avg. ppl 10.98 cum. examples 1918, speed 1599.46 words/sec, time elapsed 2803.54 sec\n",
            "epoch 27, iter 10530, avg. loss 51.85, avg. ppl 10.28 cum. examples 2078, speed 1333.28 words/sec, time elapsed 2806.21 sec\n",
            "epoch 27, iter 10540, avg. loss 52.97, avg. ppl 10.73 cum. examples 2238, speed 1239.07 words/sec, time elapsed 2809.09 sec\n",
            "epoch 27, iter 10550, avg. loss 52.81, avg. ppl 10.52 cum. examples 2398, speed 1419.02 words/sec, time elapsed 2811.62 sec\n",
            "epoch 27, iter 10560, avg. loss 49.51, avg. ppl 9.75 cum. examples 2558, speed 1316.76 words/sec, time elapsed 2814.26 sec\n",
            "epoch 27, iter 10570, avg. loss 43.16, avg. ppl 8.61 cum. examples 2718, speed 1713.73 words/sec, time elapsed 2816.14 sec\n",
            "epoch 27, iter 10580, avg. loss 48.18, avg. ppl 9.46 cum. examples 2878, speed 1413.45 words/sec, time elapsed 2818.56 sec\n",
            "epoch 27, iter 10590, avg. loss 54.88, avg. ppl 11.26 cum. examples 3038, speed 1258.59 words/sec, time elapsed 2821.44 sec\n",
            "epoch 27, iter 10600, avg. loss 51.49, avg. ppl 10.16 cum. examples 3198, speed 1507.03 words/sec, time elapsed 2823.80 sec\n",
            "epoch 27, iter 10610, avg. loss 54.13, avg. ppl 10.73 cum. examples 3358, speed 1332.02 words/sec, time elapsed 2826.54 sec\n",
            "epoch 27, iter 10620, avg. loss 56.20, avg. ppl 12.06 cum. examples 3518, speed 1193.85 words/sec, time elapsed 2829.57 sec\n",
            "epoch 27, iter 10630, avg. loss 49.31, avg. ppl 10.04 cum. examples 3678, speed 1210.78 words/sec, time elapsed 2832.39 sec\n",
            "epoch 27, iter 10640, avg. loss 48.47, avg. ppl 9.76 cum. examples 3838, speed 1369.67 words/sec, time elapsed 2834.88 sec\n",
            "epoch 27, iter 10650, avg. loss 51.45, avg. ppl 10.11 cum. examples 3998, speed 1352.82 words/sec, time elapsed 2837.51 sec\n",
            "epoch 27, iter 10660, avg. loss 56.53, avg. ppl 11.68 cum. examples 4158, speed 1043.91 words/sec, time elapsed 2841.03 sec\n",
            "epoch 27, iter 10670, avg. loss 54.03, avg. ppl 10.58 cum. examples 4318, speed 1453.04 words/sec, time elapsed 2843.56 sec\n",
            "epoch 27, iter 10680, avg. loss 62.46, avg. ppl 12.46 cum. examples 4478, speed 1422.57 words/sec, time elapsed 2846.34 sec\n",
            "epoch 27, iter 10690, avg. loss 62.07, avg. ppl 12.23 cum. examples 4638, speed 1265.06 words/sec, time elapsed 2849.48 sec\n",
            "epoch 27, iter 10700, avg. loss 50.96, avg. ppl 9.83 cum. examples 4798, speed 1587.59 words/sec, time elapsed 2851.72 sec\n",
            "epoch 27, iter 10710, avg. loss 49.15, avg. ppl 9.73 cum. examples 4958, speed 1162.25 words/sec, time elapsed 2854.70 sec\n",
            "epoch 27, iter 10720, avg. loss 40.66, avg. ppl 8.20 cum. examples 5118, speed 1514.20 words/sec, time elapsed 2856.74 sec\n",
            "epoch 27, iter 10730, avg. loss 52.78, avg. ppl 11.10 cum. examples 5278, speed 1528.79 words/sec, time elapsed 2859.03 sec\n",
            "epoch 27, iter 10740, avg. loss 52.46, avg. ppl 10.78 cum. examples 5438, speed 1280.43 words/sec, time elapsed 2861.79 sec\n",
            "epoch 27, iter 10750, avg. loss 38.97, avg. ppl 7.38 cum. examples 5598, speed 1528.12 words/sec, time elapsed 2863.83 sec\n",
            "epoch 27, iter 10760, avg. loss 49.39, avg. ppl 10.01 cum. examples 5758, speed 1445.18 words/sec, time elapsed 2866.21 sec\n",
            "epoch 27, iter 10770, avg. loss 56.30, avg. ppl 11.53 cum. examples 5918, speed 1283.47 words/sec, time elapsed 2869.08 sec\n",
            "epoch 27, iter 10780, avg. loss 50.97, avg. ppl 10.19 cum. examples 6078, speed 1223.41 words/sec, time elapsed 2871.95 sec\n",
            "epoch 27, iter 10790, avg. loss 67.31, avg. ppl 13.85 cum. examples 6238, speed 1353.45 words/sec, time elapsed 2874.98 sec\n",
            "epoch 27, iter 10800, avg. loss 55.36, avg. ppl 11.28 cum. examples 6398, speed 1542.86 words/sec, time elapsed 2877.35 sec\n",
            "epoch 27, iter 10810, avg. loss 58.47, avg. ppl 11.78 cum. examples 6558, speed 1194.48 words/sec, time elapsed 2880.52 sec\n",
            "epoch 27, iter 10820, avg. loss 53.32, avg. ppl 11.79 cum. examples 6718, speed 1381.60 words/sec, time elapsed 2883.03 sec\n",
            "epoch 28, iter 10830, avg. loss 51.36, avg. ppl 10.25 cum. examples 6876, speed 1388.73 words/sec, time elapsed 2885.54 sec\n",
            "epoch 28, iter 10840, avg. loss 46.78, avg. ppl 7.82 cum. examples 7036, speed 1639.85 words/sec, time elapsed 2887.76 sec\n",
            "epoch 28, iter 10850, avg. loss 48.09, avg. ppl 8.44 cum. examples 7196, speed 1269.28 words/sec, time elapsed 2890.60 sec\n",
            "epoch 28, iter 10860, avg. loss 47.77, avg. ppl 8.30 cum. examples 7356, speed 1360.02 words/sec, time elapsed 2893.25 sec\n",
            "epoch 28, iter 10870, avg. loss 46.33, avg. ppl 8.23 cum. examples 7516, speed 1178.00 words/sec, time elapsed 2896.24 sec\n",
            "epoch 28, iter 10880, avg. loss 44.46, avg. ppl 7.43 cum. examples 7676, speed 1705.57 words/sec, time elapsed 2898.32 sec\n",
            "epoch 28, iter 10890, avg. loss 44.11, avg. ppl 7.72 cum. examples 7836, speed 1405.48 words/sec, time elapsed 2900.78 sec\n",
            "epoch 28, iter 10900, avg. loss 40.08, avg. ppl 7.00 cum. examples 7996, speed 1670.47 words/sec, time elapsed 2902.75 sec\n",
            "epoch 28, iter 10910, avg. loss 43.40, avg. ppl 7.15 cum. examples 8156, speed 1394.55 words/sec, time elapsed 2905.28 sec\n",
            "epoch 28, iter 10920, avg. loss 40.96, avg. ppl 7.29 cum. examples 8316, speed 1421.40 words/sec, time elapsed 2907.60 sec\n",
            "epoch 28, iter 10930, avg. loss 49.30, avg. ppl 8.83 cum. examples 8476, speed 1170.56 words/sec, time elapsed 2910.70 sec\n",
            "epoch 28, iter 10940, avg. loss 45.63, avg. ppl 7.99 cum. examples 8636, speed 1384.20 words/sec, time elapsed 2913.23 sec\n",
            "epoch 28, iter 10950, avg. loss 50.44, avg. ppl 8.27 cum. examples 8796, speed 1152.25 words/sec, time elapsed 2916.55 sec\n",
            "epoch 28, iter 10960, avg. loss 39.66, avg. ppl 6.24 cum. examples 8956, speed 1441.90 words/sec, time elapsed 2918.95 sec\n",
            "epoch 28, iter 10970, avg. loss 39.92, avg. ppl 7.01 cum. examples 9116, speed 1321.95 words/sec, time elapsed 2921.44 sec\n",
            "epoch 28, iter 10980, avg. loss 50.24, avg. ppl 8.90 cum. examples 9276, speed 1155.65 words/sec, time elapsed 2924.62 sec\n",
            "epoch 28, iter 10990, avg. loss 41.37, avg. ppl 7.14 cum. examples 9436, speed 1467.35 words/sec, time elapsed 2926.91 sec\n",
            "epoch 28, iter 11000, avg. loss 53.32, avg. ppl 9.41 cum. examples 9596, speed 1290.30 words/sec, time elapsed 2929.86 sec\n",
            "epoch 28, iter 11010, avg. loss 55.45, avg. ppl 9.42 cum. examples 9756, speed 1146.80 words/sec, time elapsed 2933.31 sec\n",
            "epoch 28, iter 11020, avg. loss 50.74, avg. ppl 8.71 cum. examples 9916, speed 1350.84 words/sec, time elapsed 2936.09 sec\n",
            "epoch 28, iter 11030, avg. loss 39.38, avg. ppl 7.19 cum. examples 10076, speed 1305.89 words/sec, time elapsed 2938.53 sec\n",
            "epoch 28, iter 11040, avg. loss 43.21, avg. ppl 7.73 cum. examples 10236, speed 1214.00 words/sec, time elapsed 2941.32 sec\n",
            "epoch 28, iter 11050, avg. loss 49.93, avg. ppl 8.98 cum. examples 10396, speed 1046.91 words/sec, time elapsed 2944.80 sec\n",
            "epoch 28, iter 11060, avg. loss 35.82, avg. ppl 6.49 cum. examples 10556, speed 1552.45 words/sec, time elapsed 2946.77 sec\n",
            "epoch 28, iter 11070, avg. loss 48.44, avg. ppl 8.64 cum. examples 10716, speed 1567.69 words/sec, time elapsed 2949.06 sec\n",
            "epoch 28, iter 11080, avg. loss 36.83, avg. ppl 6.71 cum. examples 10876, speed 1464.14 words/sec, time elapsed 2951.18 sec\n",
            "epoch 28, iter 11090, avg. loss 50.14, avg. ppl 8.85 cum. examples 11036, speed 1306.32 words/sec, time elapsed 2953.99 sec\n",
            "epoch 28, iter 11100, avg. loss 44.59, avg. ppl 7.86 cum. examples 11196, speed 1465.44 words/sec, time elapsed 2956.36 sec\n",
            "epoch 28, iter 11110, avg. loss 46.10, avg. ppl 7.71 cum. examples 11356, speed 1449.24 words/sec, time elapsed 2958.85 sec\n",
            "epoch 28, iter 11120, avg. loss 48.30, avg. ppl 8.35 cum. examples 11516, speed 1488.92 words/sec, time elapsed 2961.29 sec\n",
            "epoch 28, iter 11130, avg. loss 52.02, avg. ppl 9.30 cum. examples 11676, speed 1361.79 words/sec, time elapsed 2964.03 sec\n",
            "epoch 28, iter 11140, avg. loss 51.04, avg. ppl 9.12 cum. examples 11836, speed 1533.47 words/sec, time elapsed 2966.44 sec\n",
            "epoch 28, iter 11150, avg. loss 42.34, avg. ppl 7.47 cum. examples 11996, speed 1496.15 words/sec, time elapsed 2968.69 sec\n",
            "epoch 28, iter 11160, avg. loss 47.38, avg. ppl 8.78 cum. examples 12156, speed 1418.36 words/sec, time elapsed 2971.15 sec\n",
            "epoch 28, iter 11170, avg. loss 43.64, avg. ppl 7.94 cum. examples 12316, speed 1425.59 words/sec, time elapsed 2973.52 sec\n",
            "epoch 28, iter 11180, avg. loss 56.42, avg. ppl 9.96 cum. examples 12476, speed 1212.90 words/sec, time elapsed 2976.76 sec\n",
            "epoch 28, iter 11190, avg. loss 46.54, avg. ppl 7.80 cum. examples 12636, speed 1396.97 words/sec, time elapsed 2979.35 sec\n",
            "epoch 28, iter 11200, avg. loss 40.41, avg. ppl 7.09 cum. examples 12796, speed 1465.99 words/sec, time elapsed 2981.60 sec\n",
            "epoch 28, iter 11200, cum. loss 49.06, cum. ppl 9.25 cum. examples 12796\n",
            "begin validation ...\n",
            "validation: iter 11200, dev. ppl 100.336713\n",
            "hit patience 3\n",
            "epoch 28, iter 11210, avg. loss 46.68, avg. ppl 8.32 cum. examples 160, speed 442.26 words/sec, time elapsed 2989.57 sec\n",
            "epoch 28, iter 11220, avg. loss 50.87, avg. ppl 8.54 cum. examples 320, speed 1590.67 words/sec, time elapsed 2991.96 sec\n",
            "epoch 29, iter 11230, avg. loss 47.12, avg. ppl 7.98 cum. examples 478, speed 1257.13 words/sec, time elapsed 2994.81 sec\n",
            "epoch 29, iter 11240, avg. loss 38.52, avg. ppl 5.86 cum. examples 638, speed 1273.29 words/sec, time elapsed 2997.55 sec\n",
            "epoch 29, iter 11250, avg. loss 31.64, avg. ppl 4.94 cum. examples 798, speed 1178.78 words/sec, time elapsed 3000.24 sec\n",
            "epoch 29, iter 11260, avg. loss 36.21, avg. ppl 5.48 cum. examples 958, speed 1534.70 words/sec, time elapsed 3002.45 sec\n",
            "epoch 29, iter 11270, avg. loss 32.83, avg. ppl 4.99 cum. examples 1118, speed 1591.03 words/sec, time elapsed 3004.51 sec\n",
            "epoch 29, iter 11280, avg. loss 35.89, avg. ppl 5.44 cum. examples 1278, speed 1519.08 words/sec, time elapsed 3006.74 sec\n",
            "epoch 29, iter 11290, avg. loss 41.97, avg. ppl 6.36 cum. examples 1438, speed 1493.77 words/sec, time elapsed 3009.17 sec\n",
            "epoch 29, iter 11300, avg. loss 47.32, avg. ppl 7.24 cum. examples 1598, speed 1282.91 words/sec, time elapsed 3012.15 sec\n",
            "epoch 29, iter 11310, avg. loss 35.72, avg. ppl 5.49 cum. examples 1758, speed 1478.94 words/sec, time elapsed 3014.42 sec\n",
            "epoch 29, iter 11320, avg. loss 40.13, avg. ppl 6.27 cum. examples 1918, speed 1258.84 words/sec, time elapsed 3017.20 sec\n",
            "epoch 29, iter 11330, avg. loss 41.99, avg. ppl 6.28 cum. examples 2078, speed 1471.63 words/sec, time elapsed 3019.68 sec\n",
            "epoch 29, iter 11340, avg. loss 37.96, avg. ppl 5.60 cum. examples 2238, speed 1351.36 words/sec, time elapsed 3022.29 sec\n",
            "epoch 29, iter 11350, avg. loss 43.38, avg. ppl 6.58 cum. examples 2398, speed 1302.67 words/sec, time elapsed 3025.12 sec\n",
            "epoch 29, iter 11360, avg. loss 41.45, avg. ppl 6.77 cum. examples 2558, speed 1235.54 words/sec, time elapsed 3027.93 sec\n",
            "epoch 29, iter 11370, avg. loss 44.40, avg. ppl 6.63 cum. examples 2718, speed 1450.78 words/sec, time elapsed 3030.51 sec\n",
            "epoch 29, iter 11380, avg. loss 39.60, avg. ppl 5.94 cum. examples 2878, speed 1472.66 words/sec, time elapsed 3032.93 sec\n",
            "epoch 29, iter 11390, avg. loss 41.10, avg. ppl 7.01 cum. examples 3038, speed 1234.64 words/sec, time elapsed 3035.66 sec\n",
            "epoch 29, iter 11400, avg. loss 45.15, avg. ppl 6.84 cum. examples 3198, speed 1386.64 words/sec, time elapsed 3038.37 sec\n",
            "epoch 29, iter 11410, avg. loss 45.87, avg. ppl 7.14 cum. examples 3358, speed 1316.45 words/sec, time elapsed 3041.21 sec\n",
            "epoch 29, iter 11420, avg. loss 42.18, avg. ppl 6.57 cum. examples 3518, speed 1480.90 words/sec, time elapsed 3043.63 sec\n",
            "epoch 29, iter 11430, avg. loss 48.09, avg. ppl 7.80 cum. examples 3678, speed 1298.79 words/sec, time elapsed 3046.52 sec\n",
            "epoch 29, iter 11440, avg. loss 43.77, avg. ppl 7.08 cum. examples 3838, speed 1220.14 words/sec, time elapsed 3049.45 sec\n",
            "epoch 29, iter 11450, avg. loss 33.71, avg. ppl 5.43 cum. examples 3998, speed 1556.45 words/sec, time elapsed 3051.50 sec\n",
            "epoch 29, iter 11460, avg. loss 43.60, avg. ppl 6.87 cum. examples 4158, speed 1354.51 words/sec, time elapsed 3054.17 sec\n",
            "epoch 29, iter 11470, avg. loss 45.80, avg. ppl 6.86 cum. examples 4318, speed 1269.68 words/sec, time elapsed 3057.16 sec\n",
            "epoch 29, iter 11480, avg. loss 39.52, avg. ppl 5.93 cum. examples 4478, speed 1567.25 words/sec, time elapsed 3059.43 sec\n",
            "epoch 29, iter 11490, avg. loss 42.94, avg. ppl 6.47 cum. examples 4638, speed 1372.21 words/sec, time elapsed 3062.11 sec\n",
            "epoch 29, iter 11500, avg. loss 35.69, avg. ppl 5.54 cum. examples 4798, speed 1519.84 words/sec, time elapsed 3064.31 sec\n",
            "epoch 29, iter 11510, avg. loss 33.55, avg. ppl 5.29 cum. examples 4958, speed 1427.06 words/sec, time elapsed 3066.57 sec\n",
            "epoch 29, iter 11520, avg. loss 37.20, avg. ppl 5.85 cum. examples 5118, speed 1466.08 words/sec, time elapsed 3068.86 sec\n",
            "epoch 29, iter 11530, avg. loss 46.06, avg. ppl 7.31 cum. examples 5278, speed 1412.97 words/sec, time elapsed 3071.49 sec\n",
            "epoch 29, iter 11540, avg. loss 42.34, avg. ppl 6.64 cum. examples 5438, speed 1493.61 words/sec, time elapsed 3073.88 sec\n",
            "epoch 29, iter 11550, avg. loss 42.41, avg. ppl 6.85 cum. examples 5598, speed 1357.91 words/sec, time elapsed 3076.48 sec\n",
            "epoch 29, iter 11560, avg. loss 38.51, avg. ppl 6.31 cum. examples 5758, speed 1231.43 words/sec, time elapsed 3079.19 sec\n",
            "epoch 29, iter 11570, avg. loss 44.74, avg. ppl 7.08 cum. examples 5918, speed 1117.91 words/sec, time elapsed 3082.46 sec\n",
            "epoch 29, iter 11580, avg. loss 34.47, avg. ppl 5.57 cum. examples 6078, speed 1460.31 words/sec, time elapsed 3084.66 sec\n",
            "epoch 29, iter 11590, avg. loss 41.47, avg. ppl 6.98 cum. examples 6238, speed 1130.24 words/sec, time elapsed 3087.68 sec\n",
            "epoch 29, iter 11600, avg. loss 36.29, avg. ppl 5.56 cum. examples 6398, speed 1454.44 words/sec, time elapsed 3090.01 sec\n",
            "epoch 29, iter 11610, avg. loss 40.59, avg. ppl 6.35 cum. examples 6558, speed 1569.24 words/sec, time elapsed 3092.25 sec\n",
            "epoch 29, iter 11620, avg. loss 51.54, avg. ppl 7.87 cum. examples 6718, speed 1193.45 words/sec, time elapsed 3095.60 sec\n",
            "epoch 30, iter 11630, avg. loss 42.87, avg. ppl 6.47 cum. examples 6876, speed 1443.26 words/sec, time elapsed 3098.11 sec\n",
            "epoch 30, iter 11640, avg. loss 41.42, avg. ppl 5.63 cum. examples 7036, speed 1426.49 words/sec, time elapsed 3100.80 sec\n",
            "epoch 30, iter 11650, avg. loss 28.42, avg. ppl 4.15 cum. examples 7196, speed 1479.65 words/sec, time elapsed 3102.96 sec\n",
            "epoch 30, iter 11660, avg. loss 39.46, avg. ppl 5.19 cum. examples 7356, speed 1447.85 words/sec, time elapsed 3105.61 sec\n",
            "epoch 30, iter 11670, avg. loss 37.21, avg. ppl 4.98 cum. examples 7516, speed 1528.78 words/sec, time elapsed 3108.04 sec\n",
            "epoch 30, iter 11680, avg. loss 35.36, avg. ppl 5.16 cum. examples 7676, speed 1301.73 words/sec, time elapsed 3110.69 sec\n",
            "epoch 30, iter 11690, avg. loss 36.28, avg. ppl 5.09 cum. examples 7836, speed 1385.05 words/sec, time elapsed 3113.26 sec\n",
            "epoch 30, iter 11700, avg. loss 43.01, avg. ppl 6.34 cum. examples 7996, speed 1171.04 words/sec, time elapsed 3116.44 sec\n",
            "epoch 30, iter 11710, avg. loss 38.21, avg. ppl 5.24 cum. examples 8156, speed 1224.56 words/sec, time elapsed 3119.46 sec\n",
            "epoch 30, iter 11720, avg. loss 31.14, avg. ppl 4.33 cum. examples 8316, speed 1271.18 words/sec, time elapsed 3122.13 sec\n",
            "epoch 30, iter 11730, avg. loss 37.45, avg. ppl 5.16 cum. examples 8476, speed 1455.04 words/sec, time elapsed 3124.64 sec\n",
            "epoch 30, iter 11740, avg. loss 34.74, avg. ppl 4.92 cum. examples 8636, speed 1359.46 words/sec, time elapsed 3127.21 sec\n",
            "epoch 30, iter 11750, avg. loss 33.12, avg. ppl 4.79 cum. examples 8796, speed 1660.37 words/sec, time elapsed 3129.24 sec\n",
            "epoch 30, iter 11760, avg. loss 34.01, avg. ppl 4.77 cum. examples 8956, speed 1318.91 words/sec, time elapsed 3131.88 sec\n",
            "epoch 30, iter 11770, avg. loss 35.96, avg. ppl 5.12 cum. examples 9116, speed 1221.55 words/sec, time elapsed 3134.77 sec\n",
            "epoch 30, iter 11780, avg. loss 30.72, avg. ppl 4.49 cum. examples 9276, speed 1470.85 words/sec, time elapsed 3136.99 sec\n",
            "epoch 30, iter 11790, avg. loss 35.67, avg. ppl 5.02 cum. examples 9436, speed 1448.59 words/sec, time elapsed 3139.43 sec\n",
            "epoch 30, iter 11800, avg. loss 47.52, avg. ppl 7.09 cum. examples 9596, speed 1251.51 words/sec, time elapsed 3142.54 sec\n",
            "epoch 30, iter 11810, avg. loss 36.46, avg. ppl 5.10 cum. examples 9756, speed 1372.90 words/sec, time elapsed 3145.14 sec\n",
            "epoch 30, iter 11820, avg. loss 36.61, avg. ppl 5.00 cum. examples 9916, speed 1501.82 words/sec, time elapsed 3147.57 sec\n",
            "epoch 30, iter 11830, avg. loss 32.33, avg. ppl 4.47 cum. examples 10076, speed 1514.85 words/sec, time elapsed 3149.85 sec\n",
            "epoch 30, iter 11840, avg. loss 37.61, avg. ppl 5.35 cum. examples 10236, speed 1480.46 words/sec, time elapsed 3152.27 sec\n",
            "epoch 30, iter 11850, avg. loss 39.39, avg. ppl 5.65 cum. examples 10396, speed 1179.19 words/sec, time elapsed 3155.36 sec\n",
            "epoch 30, iter 11860, avg. loss 35.21, avg. ppl 4.93 cum. examples 10556, speed 1300.64 words/sec, time elapsed 3158.07 sec\n",
            "epoch 30, iter 11870, avg. loss 35.96, avg. ppl 4.93 cum. examples 10716, speed 1266.77 words/sec, time elapsed 3160.92 sec\n",
            "epoch 30, iter 11880, avg. loss 38.95, avg. ppl 5.51 cum. examples 10876, speed 1278.76 words/sec, time elapsed 3163.78 sec\n",
            "epoch 30, iter 11890, avg. loss 34.60, avg. ppl 4.90 cum. examples 11036, speed 1365.88 words/sec, time elapsed 3166.33 sec\n",
            "epoch 30, iter 11900, avg. loss 32.62, avg. ppl 4.65 cum. examples 11196, speed 1544.62 words/sec, time elapsed 3168.52 sec\n",
            "epoch 30, iter 11910, avg. loss 36.11, avg. ppl 4.96 cum. examples 11356, speed 1544.84 words/sec, time elapsed 3170.86 sec\n",
            "epoch 30, iter 11920, avg. loss 32.54, avg. ppl 4.73 cum. examples 11516, speed 1566.88 words/sec, time elapsed 3173.00 sec\n",
            "epoch 30, iter 11930, avg. loss 28.79, avg. ppl 4.40 cum. examples 11676, speed 1365.75 words/sec, time elapsed 3175.27 sec\n",
            "epoch 30, iter 11940, avg. loss 43.60, avg. ppl 6.26 cum. examples 11836, speed 1114.53 words/sec, time elapsed 3178.68 sec\n",
            "epoch 30, iter 11950, avg. loss 35.46, avg. ppl 5.18 cum. examples 11996, speed 1344.82 words/sec, time elapsed 3181.25 sec\n",
            "epoch 30, iter 11960, avg. loss 34.73, avg. ppl 4.84 cum. examples 12156, speed 1484.45 words/sec, time elapsed 3183.62 sec\n",
            "epoch 30, iter 11970, avg. loss 36.70, avg. ppl 5.40 cum. examples 12316, speed 1181.76 words/sec, time elapsed 3186.57 sec\n",
            "epoch 30, iter 11980, avg. loss 34.15, avg. ppl 4.85 cum. examples 12476, speed 1397.35 words/sec, time elapsed 3189.05 sec\n",
            "epoch 30, iter 11990, avg. loss 37.99, avg. ppl 5.49 cum. examples 12636, speed 1433.46 words/sec, time elapsed 3191.54 sec\n",
            "epoch 30, iter 12000, avg. loss 36.23, avg. ppl 5.18 cum. examples 12796, speed 1375.38 words/sec, time elapsed 3194.10 sec\n",
            "epoch 30, iter 12000, cum. loss 38.86, cum. ppl 5.79 cum. examples 12796\n",
            "begin validation ...\n",
            "validation: iter 12000, dev. ppl 134.539358\n",
            "hit patience 4\n",
            "epoch 30, iter 12010, avg. loss 37.75, avg. ppl 5.56 cum. examples 160, speed 456.75 words/sec, time elapsed 3201.81 sec\n",
            "epoch 30, iter 12020, avg. loss 33.47, avg. ppl 4.95 cum. examples 320, speed 1502.90 words/sec, time elapsed 3204.03 sec\n",
            "epoch 30, iter 12030, avg. loss 40.28, avg. ppl 6.03 cum. examples 478, speed 1313.51 words/sec, time elapsed 3206.73 sec\n",
            "epoch 31, iter 12040, avg. loss 30.96, avg. ppl 3.96 cum. examples 638, speed 1593.21 words/sec, time elapsed 3208.99 sec\n",
            "epoch 31, iter 12050, avg. loss 29.82, avg. ppl 4.04 cum. examples 798, speed 1237.10 words/sec, time elapsed 3211.75 sec\n",
            "epoch 31, iter 12060, avg. loss 34.95, avg. ppl 4.37 cum. examples 958, speed 1483.40 words/sec, time elapsed 3214.31 sec\n",
            "epoch 31, iter 12070, avg. loss 35.67, avg. ppl 4.63 cum. examples 1118, speed 1517.72 words/sec, time elapsed 3216.76 sec\n",
            "epoch 31, iter 12080, avg. loss 26.79, avg. ppl 3.51 cum. examples 1278, speed 1330.90 words/sec, time elapsed 3219.33 sec\n",
            "epoch 31, iter 12090, avg. loss 40.98, avg. ppl 5.31 cum. examples 1438, speed 1192.82 words/sec, time elapsed 3222.62 sec\n",
            "epoch 31, iter 12100, avg. loss 29.87, avg. ppl 3.88 cum. examples 1598, speed 1333.52 words/sec, time elapsed 3225.26 sec\n",
            "epoch 31, iter 12110, avg. loss 28.85, avg. ppl 3.80 cum. examples 1758, speed 1399.10 words/sec, time elapsed 3227.73 sec\n",
            "epoch 31, iter 12120, avg. loss 27.98, avg. ppl 3.83 cum. examples 1918, speed 1521.69 words/sec, time elapsed 3229.93 sec\n",
            "epoch 31, iter 12130, avg. loss 34.81, avg. ppl 4.73 cum. examples 2078, speed 1122.88 words/sec, time elapsed 3233.12 sec\n",
            "epoch 31, iter 12140, avg. loss 33.47, avg. ppl 4.58 cum. examples 2238, speed 1229.35 words/sec, time elapsed 3235.98 sec\n",
            "epoch 31, iter 12150, avg. loss 35.68, avg. ppl 4.83 cum. examples 2398, speed 1197.00 words/sec, time elapsed 3239.00 sec\n",
            "epoch 31, iter 12160, avg. loss 33.41, avg. ppl 4.40 cum. examples 2558, speed 1163.28 words/sec, time elapsed 3242.10 sec\n",
            "epoch 31, iter 12170, avg. loss 29.58, avg. ppl 3.81 cum. examples 2718, speed 1550.17 words/sec, time elapsed 3244.39 sec\n",
            "epoch 31, iter 12180, avg. loss 29.94, avg. ppl 3.95 cum. examples 2878, speed 1357.91 words/sec, time elapsed 3246.95 sec\n",
            "epoch 31, iter 12190, avg. loss 34.52, avg. ppl 4.59 cum. examples 3038, speed 1092.27 words/sec, time elapsed 3250.27 sec\n",
            "epoch 31, iter 12200, avg. loss 29.56, avg. ppl 4.19 cum. examples 3198, speed 1161.62 words/sec, time elapsed 3253.12 sec\n",
            "epoch 31, iter 12210, avg. loss 32.85, avg. ppl 4.20 cum. examples 3358, speed 1332.93 words/sec, time elapsed 3255.86 sec\n",
            "epoch 31, iter 12220, avg. loss 34.41, avg. ppl 4.38 cum. examples 3518, speed 1431.31 words/sec, time elapsed 3258.47 sec\n",
            "epoch 31, iter 12230, avg. loss 31.65, avg. ppl 4.10 cum. examples 3678, speed 1264.05 words/sec, time elapsed 3261.31 sec\n",
            "epoch 31, iter 12240, avg. loss 29.43, avg. ppl 3.87 cum. examples 3838, speed 1462.79 words/sec, time elapsed 3263.69 sec\n",
            "epoch 31, iter 12250, avg. loss 29.00, avg. ppl 4.00 cum. examples 3998, speed 1322.14 words/sec, time elapsed 3266.22 sec\n",
            "epoch 31, iter 12260, avg. loss 33.23, avg. ppl 4.44 cum. examples 4158, speed 1522.10 words/sec, time elapsed 3268.57 sec\n",
            "epoch 31, iter 12270, avg. loss 38.35, avg. ppl 4.87 cum. examples 4318, speed 1552.26 words/sec, time elapsed 3271.06 sec\n",
            "epoch 31, iter 12280, avg. loss 28.31, avg. ppl 3.93 cum. examples 4478, speed 1314.78 words/sec, time elapsed 3273.58 sec\n",
            "epoch 31, iter 12290, avg. loss 32.71, avg. ppl 4.26 cum. examples 4638, speed 1395.63 words/sec, time elapsed 3276.17 sec\n",
            "epoch 31, iter 12300, avg. loss 30.21, avg. ppl 4.06 cum. examples 4798, speed 1458.88 words/sec, time elapsed 3278.53 sec\n",
            "epoch 31, iter 12310, avg. loss 26.05, avg. ppl 3.60 cum. examples 4958, speed 1559.76 words/sec, time elapsed 3280.62 sec\n",
            "epoch 31, iter 12320, avg. loss 36.78, avg. ppl 4.94 cum. examples 5118, speed 1276.06 words/sec, time elapsed 3283.50 sec\n",
            "epoch 31, iter 12330, avg. loss 32.26, avg. ppl 4.35 cum. examples 5278, speed 1317.64 words/sec, time elapsed 3286.17 sec\n",
            "epoch 31, iter 12340, avg. loss 32.88, avg. ppl 4.25 cum. examples 5438, speed 1544.46 words/sec, time elapsed 3288.52 sec\n",
            "epoch 31, iter 12350, avg. loss 33.72, avg. ppl 4.62 cum. examples 5598, speed 1388.87 words/sec, time elapsed 3291.06 sec\n",
            "epoch 31, iter 12360, avg. loss 29.96, avg. ppl 4.15 cum. examples 5758, speed 1305.57 words/sec, time elapsed 3293.64 sec\n",
            "epoch 31, iter 12370, avg. loss 31.34, avg. ppl 4.15 cum. examples 5918, speed 1418.09 words/sec, time elapsed 3296.13 sec\n",
            "epoch 31, iter 12380, avg. loss 31.18, avg. ppl 3.96 cum. examples 6078, speed 1477.83 words/sec, time elapsed 3298.58 sec\n",
            "epoch 31, iter 12390, avg. loss 32.87, avg. ppl 4.34 cum. examples 6238, speed 1389.63 words/sec, time elapsed 3301.15 sec\n",
            "epoch 31, iter 12400, avg. loss 32.11, avg. ppl 4.60 cum. examples 6398, speed 1220.87 words/sec, time elapsed 3303.91 sec\n",
            "epoch 31, iter 12410, avg. loss 35.47, avg. ppl 4.88 cum. examples 6558, speed 1423.10 words/sec, time elapsed 3306.43 sec\n",
            "epoch 31, iter 12420, avg. loss 30.45, avg. ppl 4.24 cum. examples 6718, speed 1472.33 words/sec, time elapsed 3308.72 sec\n",
            "epoch 31, iter 12430, avg. loss 26.96, avg. ppl 3.63 cum. examples 6878, speed 1530.74 words/sec, time elapsed 3310.91 sec\n",
            "epoch 32, iter 12440, avg. loss 29.72, avg. ppl 3.81 cum. examples 7036, speed 1263.52 words/sec, time elapsed 3313.68 sec\n",
            "epoch 32, iter 12450, avg. loss 24.42, avg. ppl 3.16 cum. examples 7196, speed 1336.00 words/sec, time elapsed 3316.22 sec\n",
            "epoch 32, iter 12460, avg. loss 33.09, avg. ppl 4.04 cum. examples 7356, speed 1284.69 words/sec, time elapsed 3319.17 sec\n",
            "epoch 32, iter 12470, avg. loss 31.59, avg. ppl 3.83 cum. examples 7516, speed 1153.29 words/sec, time elapsed 3322.44 sec\n",
            "epoch 32, iter 12480, avg. loss 28.87, avg. ppl 3.54 cum. examples 7676, speed 1274.28 words/sec, time elapsed 3325.30 sec\n",
            "epoch 32, iter 12490, avg. loss 25.08, avg. ppl 3.43 cum. examples 7836, speed 1351.16 words/sec, time elapsed 3327.71 sec\n",
            "epoch 32, iter 12500, avg. loss 28.93, avg. ppl 3.52 cum. examples 7996, speed 1539.02 words/sec, time elapsed 3330.10 sec\n",
            "epoch 32, iter 12510, avg. loss 24.98, avg. ppl 3.07 cum. examples 8156, speed 1674.92 words/sec, time elapsed 3332.22 sec\n",
            "epoch 32, iter 12520, avg. loss 32.47, avg. ppl 4.01 cum. examples 8316, speed 1154.44 words/sec, time elapsed 3335.46 sec\n",
            "epoch 32, iter 12530, avg. loss 23.49, avg. ppl 3.04 cum. examples 8476, speed 1557.16 words/sec, time elapsed 3337.64 sec\n",
            "epoch 32, iter 12540, avg. loss 26.21, avg. ppl 3.33 cum. examples 8636, speed 1488.43 words/sec, time elapsed 3339.98 sec\n",
            "epoch 32, iter 12550, avg. loss 24.81, avg. ppl 3.26 cum. examples 8796, speed 1475.70 words/sec, time elapsed 3342.26 sec\n",
            "epoch 32, iter 12560, avg. loss 33.26, avg. ppl 4.02 cum. examples 8956, speed 1342.43 words/sec, time elapsed 3345.11 sec\n",
            "epoch 32, iter 12570, avg. loss 26.96, avg. ppl 3.36 cum. examples 9116, speed 1470.58 words/sec, time elapsed 3347.53 sec\n",
            "epoch 32, iter 12580, avg. loss 24.22, avg. ppl 3.17 cum. examples 9276, speed 1422.21 words/sec, time elapsed 3349.89 sec\n",
            "epoch 32, iter 12590, avg. loss 35.63, avg. ppl 4.36 cum. examples 9436, speed 1211.25 words/sec, time elapsed 3353.08 sec\n",
            "epoch 32, iter 12600, avg. loss 30.21, avg. ppl 3.80 cum. examples 9596, speed 1255.36 words/sec, time elapsed 3355.96 sec\n",
            "epoch 32, iter 12610, avg. loss 27.91, avg. ppl 3.61 cum. examples 9756, speed 1289.11 words/sec, time elapsed 3358.66 sec\n",
            "epoch 32, iter 12620, avg. loss 27.61, avg. ppl 3.45 cum. examples 9916, speed 1354.07 words/sec, time elapsed 3361.30 sec\n",
            "epoch 32, iter 12630, avg. loss 27.10, avg. ppl 3.42 cum. examples 10076, speed 1276.42 words/sec, time elapsed 3364.06 sec\n",
            "epoch 32, iter 12640, avg. loss 25.94, avg. ppl 3.26 cum. examples 10236, speed 1494.94 words/sec, time elapsed 3366.41 sec\n",
            "epoch 32, iter 12650, avg. loss 26.23, avg. ppl 3.37 cum. examples 10396, speed 1385.57 words/sec, time elapsed 3368.90 sec\n",
            "epoch 32, iter 12660, avg. loss 25.76, avg. ppl 3.43 cum. examples 10556, speed 1394.06 words/sec, time elapsed 3371.30 sec\n",
            "epoch 32, iter 12670, avg. loss 25.12, avg. ppl 3.18 cum. examples 10716, speed 1528.69 words/sec, time elapsed 3373.58 sec\n",
            "epoch 32, iter 12680, avg. loss 21.91, avg. ppl 3.02 cum. examples 10876, speed 1472.34 words/sec, time elapsed 3375.73 sec\n",
            "epoch 32, iter 12690, avg. loss 33.88, avg. ppl 4.33 cum. examples 11036, speed 1215.61 words/sec, time elapsed 3378.77 sec\n",
            "epoch 32, iter 12700, avg. loss 19.55, avg. ppl 2.79 cum. examples 11196, speed 1466.34 words/sec, time elapsed 3380.86 sec\n",
            "epoch 32, iter 12710, avg. loss 24.62, avg. ppl 3.33 cum. examples 11356, speed 1368.37 words/sec, time elapsed 3383.25 sec\n",
            "epoch 32, iter 12720, avg. loss 32.23, avg. ppl 4.01 cum. examples 11516, speed 1421.33 words/sec, time elapsed 3385.87 sec\n",
            "epoch 32, iter 12730, avg. loss 26.13, avg. ppl 3.46 cum. examples 11676, speed 1211.57 words/sec, time elapsed 3388.64 sec\n",
            "epoch 32, iter 12740, avg. loss 30.48, avg. ppl 3.91 cum. examples 11836, speed 1197.62 words/sec, time elapsed 3391.63 sec\n",
            "epoch 32, iter 12750, avg. loss 25.91, avg. ppl 3.46 cum. examples 11996, speed 1222.86 words/sec, time elapsed 3394.37 sec\n",
            "epoch 32, iter 12760, avg. loss 34.64, avg. ppl 4.30 cum. examples 12156, speed 1199.91 words/sec, time elapsed 3397.53 sec\n",
            "epoch 32, iter 12770, avg. loss 22.75, avg. ppl 3.06 cum. examples 12316, speed 1389.85 words/sec, time elapsed 3399.87 sec\n",
            "epoch 32, iter 12780, avg. loss 25.96, avg. ppl 3.34 cum. examples 12476, speed 1273.10 words/sec, time elapsed 3402.58 sec\n",
            "epoch 32, iter 12790, avg. loss 33.50, avg. ppl 3.96 cum. examples 12636, speed 1385.44 words/sec, time elapsed 3405.39 sec\n",
            "epoch 32, iter 12800, avg. loss 31.52, avg. ppl 3.82 cum. examples 12796, speed 1493.23 words/sec, time elapsed 3407.91 sec\n",
            "epoch 32, iter 12800, cum. loss 30.29, cum. ppl 3.95 cum. examples 12796\n",
            "begin validation ...\n",
            "validation: iter 12800, dev. ppl 172.997588\n",
            "hit patience 5\n",
            "hit #3 trial\n",
            "load previously best model and decay learning rate to 0.000125\n",
            "restore parameters of the optimizers\n",
            "epoch 32, iter 12810, avg. loss 93.61, avg. ppl 44.81 cum. examples 160, speed 493.84 words/sec, time elapsed 3415.89 sec\n",
            "epoch 32, iter 12820, avg. loss 79.58, avg. ppl 40.29 cum. examples 320, speed 1365.47 words/sec, time elapsed 3418.41 sec\n",
            "epoch 32, iter 12830, avg. loss 81.11, avg. ppl 39.95 cum. examples 480, speed 1444.77 words/sec, time elapsed 3420.84 sec\n",
            "epoch 33, iter 12840, avg. loss 73.07, avg. ppl 36.85 cum. examples 638, speed 1373.98 words/sec, time elapsed 3423.17 sec\n",
            "epoch 33, iter 12850, avg. loss 73.28, avg. ppl 35.42 cum. examples 798, speed 1518.14 words/sec, time elapsed 3425.34 sec\n",
            "epoch 33, iter 12860, avg. loss 72.30, avg. ppl 33.65 cum. examples 958, speed 1381.15 words/sec, time elapsed 3427.72 sec\n",
            "epoch 33, iter 12870, avg. loss 81.26, avg. ppl 37.89 cum. examples 1118, speed 1306.63 words/sec, time elapsed 3430.46 sec\n",
            "epoch 33, iter 12880, avg. loss 82.42, avg. ppl 39.39 cum. examples 1278, speed 1390.24 words/sec, time elapsed 3433.04 sec\n",
            "epoch 33, iter 12890, avg. loss 85.74, avg. ppl 37.72 cum. examples 1438, speed 1343.99 words/sec, time elapsed 3435.85 sec\n",
            "epoch 33, iter 12900, avg. loss 80.92, avg. ppl 38.96 cum. examples 1598, speed 1367.45 words/sec, time elapsed 3438.44 sec\n",
            "epoch 33, iter 12910, avg. loss 77.10, avg. ppl 33.53 cum. examples 1758, speed 1327.31 words/sec, time elapsed 3441.08 sec\n",
            "epoch 33, iter 12920, avg. loss 74.12, avg. ppl 33.40 cum. examples 1918, speed 1303.68 words/sec, time elapsed 3443.68 sec\n",
            "epoch 33, iter 12930, avg. loss 74.74, avg. ppl 35.24 cum. examples 2078, speed 1315.54 words/sec, time elapsed 3446.23 sec\n",
            "epoch 33, iter 12940, avg. loss 80.19, avg. ppl 37.34 cum. examples 2238, speed 1259.31 words/sec, time elapsed 3449.04 sec\n",
            "epoch 33, iter 12950, avg. loss 75.71, avg. ppl 34.75 cum. examples 2398, speed 1443.90 words/sec, time elapsed 3451.41 sec\n",
            "epoch 33, iter 12960, avg. loss 85.19, avg. ppl 38.01 cum. examples 2558, speed 1336.38 words/sec, time elapsed 3454.21 sec\n",
            "epoch 33, iter 12970, avg. loss 80.25, avg. ppl 36.43 cum. examples 2718, speed 1332.61 words/sec, time elapsed 3456.89 sec\n",
            "epoch 33, iter 12980, avg. loss 76.29, avg. ppl 34.83 cum. examples 2878, speed 1551.69 words/sec, time elapsed 3459.11 sec\n",
            "epoch 33, iter 12990, avg. loss 81.25, avg. ppl 35.70 cum. examples 3038, speed 1352.95 words/sec, time elapsed 3461.79 sec\n",
            "epoch 33, iter 13000, avg. loss 79.48, avg. ppl 36.32 cum. examples 3198, speed 1343.29 words/sec, time elapsed 3464.43 sec\n",
            "epoch 33, iter 13010, avg. loss 82.37, avg. ppl 35.68 cum. examples 3358, speed 1355.77 words/sec, time elapsed 3467.15 sec\n",
            "epoch 33, iter 13020, avg. loss 69.26, avg. ppl 28.91 cum. examples 3518, speed 1640.94 words/sec, time elapsed 3469.16 sec\n",
            "epoch 33, iter 13030, avg. loss 84.63, avg. ppl 34.34 cum. examples 3678, speed 1395.27 words/sec, time elapsed 3471.90 sec\n",
            "epoch 33, iter 13040, avg. loss 78.06, avg. ppl 35.28 cum. examples 3838, speed 1476.40 words/sec, time elapsed 3474.28 sec\n",
            "epoch 33, iter 13050, avg. loss 82.61, avg. ppl 34.92 cum. examples 3998, speed 1449.45 words/sec, time elapsed 3476.84 sec\n",
            "epoch 33, iter 13060, avg. loss 80.42, avg. ppl 33.35 cum. examples 4158, speed 1388.77 words/sec, time elapsed 3479.48 sec\n",
            "epoch 33, iter 13070, avg. loss 82.26, avg. ppl 37.48 cum. examples 4318, speed 1150.29 words/sec, time elapsed 3482.64 sec\n",
            "epoch 33, iter 13080, avg. loss 82.87, avg. ppl 36.85 cum. examples 4478, speed 1213.10 words/sec, time elapsed 3485.67 sec\n",
            "epoch 33, iter 13090, avg. loss 78.05, avg. ppl 34.71 cum. examples 4638, speed 1273.56 words/sec, time elapsed 3488.44 sec\n",
            "epoch 33, iter 13100, avg. loss 73.25, avg. ppl 32.23 cum. examples 4798, speed 1233.37 words/sec, time elapsed 3491.17 sec\n",
            "epoch 33, iter 13110, avg. loss 80.56, avg. ppl 35.74 cum. examples 4958, speed 1471.69 words/sec, time elapsed 3493.62 sec\n",
            "epoch 33, iter 13120, avg. loss 79.13, avg. ppl 36.48 cum. examples 5118, speed 1229.23 words/sec, time elapsed 3496.49 sec\n",
            "epoch 33, iter 13130, avg. loss 87.43, avg. ppl 37.07 cum. examples 5278, speed 1196.00 words/sec, time elapsed 3499.72 sec\n",
            "epoch 33, iter 13140, avg. loss 75.66, avg. ppl 32.71 cum. examples 5438, speed 1316.41 words/sec, time elapsed 3502.36 sec\n",
            "epoch 33, iter 13150, avg. loss 75.66, avg. ppl 32.47 cum. examples 5598, speed 1463.78 words/sec, time elapsed 3504.74 sec\n",
            "epoch 33, iter 13160, avg. loss 75.33, avg. ppl 35.07 cum. examples 5758, speed 1456.79 words/sec, time elapsed 3507.06 sec\n",
            "epoch 33, iter 13170, avg. loss 77.50, avg. ppl 31.39 cum. examples 5918, speed 1426.25 words/sec, time elapsed 3509.58 sec\n",
            "epoch 33, iter 13180, avg. loss 77.06, avg. ppl 32.24 cum. examples 6078, speed 1439.61 words/sec, time elapsed 3512.05 sec\n",
            "epoch 33, iter 13190, avg. loss 75.92, avg. ppl 32.74 cum. examples 6238, speed 1502.63 words/sec, time elapsed 3514.37 sec\n",
            "epoch 33, iter 13200, avg. loss 88.07, avg. ppl 37.16 cum. examples 6398, speed 1280.80 words/sec, time elapsed 3517.41 sec\n",
            "epoch 33, iter 13210, avg. loss 72.18, avg. ppl 30.63 cum. examples 6558, speed 1467.35 words/sec, time elapsed 3519.71 sec\n",
            "epoch 33, iter 13220, avg. loss 75.81, avg. ppl 31.10 cum. examples 6718, speed 1358.02 words/sec, time elapsed 3522.31 sec\n",
            "epoch 33, iter 13230, avg. loss 77.00, avg. ppl 33.65 cum. examples 6878, speed 1352.52 words/sec, time elapsed 3524.90 sec\n",
            "epoch 34, iter 13240, avg. loss 65.96, avg. ppl 27.69 cum. examples 7036, speed 1293.21 words/sec, time elapsed 3527.33 sec\n",
            "epoch 34, iter 13250, avg. loss 73.02, avg. ppl 27.93 cum. examples 7196, speed 1399.24 words/sec, time elapsed 3529.83 sec\n",
            "epoch 34, iter 13260, avg. loss 70.46, avg. ppl 27.73 cum. examples 7356, speed 1706.37 words/sec, time elapsed 3531.82 sec\n",
            "epoch 34, iter 13270, avg. loss 74.72, avg. ppl 29.18 cum. examples 7516, speed 1260.85 words/sec, time elapsed 3534.63 sec\n",
            "epoch 34, iter 13280, avg. loss 67.62, avg. ppl 27.41 cum. examples 7676, speed 1286.78 words/sec, time elapsed 3537.17 sec\n",
            "epoch 34, iter 13290, avg. loss 77.33, avg. ppl 28.46 cum. examples 7836, speed 1419.41 words/sec, time elapsed 3539.78 sec\n",
            "epoch 34, iter 13300, avg. loss 75.16, avg. ppl 28.57 cum. examples 7996, speed 1378.27 words/sec, time elapsed 3542.38 sec\n",
            "epoch 34, iter 13310, avg. loss 67.10, avg. ppl 25.06 cum. examples 8156, speed 1564.19 words/sec, time elapsed 3544.51 sec\n",
            "epoch 34, iter 13320, avg. loss 79.25, avg. ppl 32.20 cum. examples 8316, speed 1184.57 words/sec, time elapsed 3547.59 sec\n",
            "epoch 34, iter 13330, avg. loss 67.40, avg. ppl 25.57 cum. examples 8476, speed 1401.46 words/sec, time elapsed 3549.97 sec\n",
            "epoch 34, iter 13340, avg. loss 71.05, avg. ppl 27.06 cum. examples 8636, speed 1499.83 words/sec, time elapsed 3552.27 sec\n",
            "epoch 34, iter 13350, avg. loss 84.23, avg. ppl 32.66 cum. examples 8796, speed 1448.14 words/sec, time elapsed 3554.94 sec\n",
            "epoch 34, iter 13360, avg. loss 71.05, avg. ppl 25.38 cum. examples 8956, speed 1491.02 words/sec, time elapsed 3557.29 sec\n",
            "epoch 34, iter 13370, avg. loss 76.02, avg. ppl 26.91 cum. examples 9116, speed 1319.65 words/sec, time elapsed 3560.09 sec\n",
            "epoch 34, iter 13380, avg. loss 69.95, avg. ppl 25.93 cum. examples 9276, speed 1389.36 words/sec, time elapsed 3562.57 sec\n",
            "epoch 34, iter 13390, avg. loss 75.49, avg. ppl 28.54 cum. examples 9436, speed 1373.66 words/sec, time elapsed 3565.19 sec\n",
            "epoch 34, iter 13400, avg. loss 74.70, avg. ppl 28.26 cum. examples 9596, speed 1383.52 words/sec, time elapsed 3567.78 sec\n",
            "epoch 34, iter 13410, avg. loss 58.22, avg. ppl 23.54 cum. examples 9756, speed 1346.39 words/sec, time elapsed 3569.97 sec\n",
            "epoch 34, iter 13420, avg. loss 71.87, avg. ppl 27.23 cum. examples 9916, speed 1374.99 words/sec, time elapsed 3572.50 sec\n",
            "epoch 34, iter 13430, avg. loss 68.00, avg. ppl 25.86 cum. examples 10076, speed 1480.00 words/sec, time elapsed 3574.76 sec\n",
            "epoch 34, iter 13440, avg. loss 69.81, avg. ppl 26.74 cum. examples 10236, speed 1342.99 words/sec, time elapsed 3577.29 sec\n",
            "epoch 34, iter 13450, avg. loss 71.45, avg. ppl 26.76 cum. examples 10396, speed 1409.34 words/sec, time elapsed 3579.76 sec\n",
            "epoch 34, iter 13460, avg. loss 75.31, avg. ppl 30.64 cum. examples 10556, speed 1306.76 words/sec, time elapsed 3582.45 sec\n",
            "epoch 34, iter 13470, avg. loss 69.77, avg. ppl 27.11 cum. examples 10716, speed 1538.71 words/sec, time elapsed 3584.65 sec\n",
            "epoch 34, iter 13480, avg. loss 75.50, avg. ppl 29.04 cum. examples 10876, speed 1324.68 words/sec, time elapsed 3587.36 sec\n",
            "epoch 34, iter 13490, avg. loss 72.05, avg. ppl 27.88 cum. examples 11036, speed 1431.44 words/sec, time elapsed 3589.78 sec\n",
            "epoch 34, iter 13500, avg. loss 71.50, avg. ppl 26.01 cum. examples 11196, speed 1270.63 words/sec, time elapsed 3592.54 sec\n",
            "epoch 34, iter 13510, avg. loss 69.43, avg. ppl 26.45 cum. examples 11356, speed 1321.87 words/sec, time elapsed 3595.11 sec\n",
            "epoch 34, iter 13520, avg. loss 68.96, avg. ppl 26.76 cum. examples 11516, speed 1356.02 words/sec, time elapsed 3597.58 sec\n",
            "epoch 34, iter 13530, avg. loss 80.76, avg. ppl 32.59 cum. examples 11676, speed 1248.08 words/sec, time elapsed 3600.55 sec\n",
            "epoch 34, iter 13540, avg. loss 76.38, avg. ppl 29.77 cum. examples 11836, speed 1242.39 words/sec, time elapsed 3603.45 sec\n",
            "epoch 34, iter 13550, avg. loss 77.85, avg. ppl 29.11 cum. examples 11996, speed 1253.90 words/sec, time elapsed 3606.40 sec\n",
            "epoch 34, iter 13560, avg. loss 83.60, avg. ppl 29.51 cum. examples 12156, speed 1333.62 words/sec, time elapsed 3609.36 sec\n",
            "epoch 34, iter 13570, avg. loss 73.13, avg. ppl 26.91 cum. examples 12316, speed 1428.30 words/sec, time elapsed 3611.85 sec\n",
            "epoch 34, iter 13580, avg. loss 80.65, avg. ppl 30.24 cum. examples 12476, speed 1234.45 words/sec, time elapsed 3614.92 sec\n",
            "epoch 34, iter 13590, avg. loss 77.42, avg. ppl 29.92 cum. examples 12636, speed 1177.09 words/sec, time elapsed 3618.01 sec\n",
            "epoch 34, iter 13600, avg. loss 70.22, avg. ppl 25.26 cum. examples 12796, speed 1428.00 words/sec, time elapsed 3620.45 sec\n",
            "epoch 34, iter 13600, cum. loss 76.27, cum. ppl 31.70 cum. examples 12796\n",
            "begin validation ...\n",
            "validation: iter 13600, dev. ppl 57.331584\n",
            "hit patience 1\n",
            "epoch 34, iter 13610, avg. loss 79.20, avg. ppl 28.52 cum. examples 160, speed 481.90 words/sec, time elapsed 3628.30 sec\n",
            "epoch 34, iter 13620, avg. loss 84.90, avg. ppl 30.78 cum. examples 320, speed 1129.59 words/sec, time elapsed 3631.81 sec\n",
            "epoch 34, iter 13630, avg. loss 76.95, avg. ppl 28.96 cum. examples 480, speed 1298.05 words/sec, time elapsed 3634.63 sec\n",
            "epoch 35, iter 13640, avg. loss 68.81, avg. ppl 24.61 cum. examples 638, speed 1369.92 words/sec, time elapsed 3637.10 sec\n",
            "epoch 35, iter 13650, avg. loss 69.12, avg. ppl 24.63 cum. examples 798, speed 1308.09 words/sec, time elapsed 3639.74 sec\n",
            "epoch 35, iter 13660, avg. loss 69.29, avg. ppl 25.05 cum. examples 958, speed 1313.83 words/sec, time elapsed 3642.36 sec\n",
            "epoch 35, iter 13670, avg. loss 71.41, avg. ppl 24.05 cum. examples 1118, speed 1201.43 words/sec, time elapsed 3645.35 sec\n",
            "epoch 35, iter 13680, avg. loss 67.50, avg. ppl 21.76 cum. examples 1278, speed 1297.07 words/sec, time elapsed 3648.06 sec\n",
            "epoch 35, iter 13690, avg. loss 79.34, avg. ppl 26.33 cum. examples 1438, speed 1322.48 words/sec, time elapsed 3650.99 sec\n",
            "epoch 35, iter 13700, avg. loss 65.45, avg. ppl 22.10 cum. examples 1598, speed 1476.50 words/sec, time elapsed 3653.28 sec\n",
            "epoch 35, iter 13710, avg. loss 73.56, avg. ppl 24.92 cum. examples 1758, speed 1497.86 words/sec, time elapsed 3655.73 sec\n",
            "epoch 35, iter 13720, avg. loss 69.78, avg. ppl 23.18 cum. examples 1918, speed 1358.93 words/sec, time elapsed 3658.34 sec\n",
            "epoch 35, iter 13730, avg. loss 64.55, avg. ppl 20.53 cum. examples 2078, speed 1211.86 words/sec, time elapsed 3661.16 sec\n",
            "epoch 35, iter 13740, avg. loss 68.44, avg. ppl 23.75 cum. examples 2238, speed 1379.58 words/sec, time elapsed 3663.67 sec\n",
            "epoch 35, iter 13750, avg. loss 66.67, avg. ppl 20.96 cum. examples 2398, speed 1329.41 words/sec, time elapsed 3666.30 sec\n",
            "epoch 35, iter 13760, avg. loss 74.57, avg. ppl 24.89 cum. examples 2558, speed 1395.42 words/sec, time elapsed 3668.96 sec\n",
            "epoch 35, iter 13770, avg. loss 70.67, avg. ppl 22.42 cum. examples 2718, speed 1376.79 words/sec, time elapsed 3671.60 sec\n",
            "epoch 35, iter 13780, avg. loss 69.92, avg. ppl 23.51 cum. examples 2878, speed 1469.21 words/sec, time elapsed 3674.02 sec\n",
            "epoch 35, iter 13790, avg. loss 61.60, avg. ppl 20.02 cum. examples 3038, speed 1433.26 words/sec, time elapsed 3676.31 sec\n",
            "epoch 35, iter 13800, avg. loss 69.03, avg. ppl 22.32 cum. examples 3198, speed 1357.91 words/sec, time elapsed 3678.93 sec\n",
            "epoch 35, iter 13810, avg. loss 66.11, avg. ppl 22.02 cum. examples 3358, speed 1410.64 words/sec, time elapsed 3681.35 sec\n",
            "epoch 35, iter 13820, avg. loss 73.51, avg. ppl 23.45 cum. examples 3518, speed 1217.11 words/sec, time elapsed 3684.42 sec\n",
            "epoch 35, iter 13830, avg. loss 74.53, avg. ppl 24.87 cum. examples 3678, speed 1293.52 words/sec, time elapsed 3687.29 sec\n",
            "epoch 35, iter 13840, avg. loss 71.34, avg. ppl 23.57 cum. examples 3838, speed 1500.83 words/sec, time elapsed 3689.69 sec\n",
            "epoch 35, iter 13850, avg. loss 71.76, avg. ppl 23.83 cum. examples 3998, speed 1271.95 words/sec, time elapsed 3692.54 sec\n",
            "epoch 35, iter 13860, avg. loss 65.25, avg. ppl 21.61 cum. examples 4158, speed 1341.69 words/sec, time elapsed 3695.07 sec\n",
            "epoch 35, iter 13870, avg. loss 63.70, avg. ppl 21.14 cum. examples 4318, speed 1189.40 words/sec, time elapsed 3697.88 sec\n",
            "epoch 35, iter 13880, avg. loss 72.06, avg. ppl 25.20 cum. examples 4478, speed 1604.21 words/sec, time elapsed 3700.11 sec\n",
            "epoch 35, iter 13890, avg. loss 69.64, avg. ppl 23.02 cum. examples 4638, speed 1115.00 words/sec, time elapsed 3703.29 sec\n",
            "epoch 35, iter 13900, avg. loss 60.74, avg. ppl 20.02 cum. examples 4798, speed 1654.53 words/sec, time elapsed 3705.25 sec\n",
            "epoch 35, iter 13910, avg. loss 74.81, avg. ppl 24.19 cum. examples 4958, speed 1396.33 words/sec, time elapsed 3707.95 sec\n",
            "epoch 35, iter 13920, avg. loss 75.63, avg. ppl 26.21 cum. examples 5118, speed 1294.12 words/sec, time elapsed 3710.81 sec\n",
            "epoch 35, iter 13930, avg. loss 61.35, avg. ppl 18.57 cum. examples 5278, speed 1438.20 words/sec, time elapsed 3713.14 sec\n",
            "epoch 35, iter 13940, avg. loss 62.57, avg. ppl 21.38 cum. examples 5438, speed 1429.30 words/sec, time elapsed 3715.43 sec\n",
            "epoch 35, iter 13950, avg. loss 67.07, avg. ppl 22.31 cum. examples 5598, speed 1605.69 words/sec, time elapsed 3717.58 sec\n",
            "epoch 35, iter 13960, avg. loss 71.48, avg. ppl 23.13 cum. examples 5758, speed 1207.40 words/sec, time elapsed 3720.60 sec\n",
            "epoch 35, iter 13970, avg. loss 73.45, avg. ppl 24.98 cum. examples 5918, speed 1350.43 words/sec, time elapsed 3723.30 sec\n",
            "epoch 35, iter 13980, avg. loss 70.81, avg. ppl 24.17 cum. examples 6078, speed 1406.99 words/sec, time elapsed 3725.83 sec\n",
            "epoch 35, iter 13990, avg. loss 85.76, avg. ppl 29.65 cum. examples 6238, speed 1509.87 words/sec, time elapsed 3728.51 sec\n",
            "epoch 35, iter 14000, avg. loss 70.40, avg. ppl 22.69 cum. examples 6398, speed 1192.69 words/sec, time elapsed 3731.54 sec\n",
            "epoch 35, iter 14010, avg. loss 71.76, avg. ppl 22.76 cum. examples 6558, speed 1579.01 words/sec, time elapsed 3733.87 sec\n",
            "epoch 35, iter 14020, avg. loss 59.88, avg. ppl 19.82 cum. examples 6718, speed 1637.31 words/sec, time elapsed 3735.82 sec\n",
            "epoch 35, iter 14030, avg. loss 67.93, avg. ppl 23.60 cum. examples 6878, speed 1271.41 words/sec, time elapsed 3738.53 sec\n",
            "epoch 36, iter 14040, avg. loss 65.20, avg. ppl 20.82 cum. examples 7036, speed 1287.29 words/sec, time elapsed 3741.16 sec\n",
            "epoch 36, iter 14050, avg. loss 73.19, avg. ppl 20.45 cum. examples 7196, speed 1320.01 words/sec, time elapsed 3744.10 sec\n",
            "epoch 36, iter 14060, avg. loss 76.39, avg. ppl 22.58 cum. examples 7356, speed 1293.28 words/sec, time elapsed 3747.14 sec\n",
            "epoch 36, iter 14070, avg. loss 68.77, avg. ppl 19.97 cum. examples 7516, speed 1386.29 words/sec, time elapsed 3749.79 sec\n",
            "epoch 36, iter 14080, avg. loss 65.30, avg. ppl 18.04 cum. examples 7676, speed 1450.04 words/sec, time elapsed 3752.28 sec\n",
            "epoch 36, iter 14090, avg. loss 62.95, avg. ppl 17.09 cum. examples 7836, speed 1292.03 words/sec, time elapsed 3755.02 sec\n",
            "epoch 36, iter 14100, avg. loss 65.77, avg. ppl 18.42 cum. examples 7996, speed 1274.62 words/sec, time elapsed 3757.86 sec\n",
            "epoch 36, iter 14110, avg. loss 58.96, avg. ppl 17.15 cum. examples 8156, speed 1504.41 words/sec, time elapsed 3760.06 sec\n",
            "epoch 36, iter 14120, avg. loss 59.83, avg. ppl 16.88 cum. examples 8316, speed 1443.06 words/sec, time elapsed 3762.41 sec\n",
            "epoch 36, iter 14130, avg. loss 70.63, avg. ppl 21.08 cum. examples 8476, speed 1413.50 words/sec, time elapsed 3765.03 sec\n",
            "epoch 36, iter 14140, avg. loss 60.03, avg. ppl 18.01 cum. examples 8636, speed 1424.01 words/sec, time elapsed 3767.37 sec\n",
            "epoch 36, iter 14150, avg. loss 61.87, avg. ppl 17.41 cum. examples 8796, speed 1620.13 words/sec, time elapsed 3769.51 sec\n",
            "epoch 36, iter 14160, avg. loss 70.80, avg. ppl 21.76 cum. examples 8956, speed 1172.53 words/sec, time elapsed 3772.64 sec\n",
            "epoch 36, iter 14170, avg. loss 58.50, avg. ppl 17.23 cum. examples 9116, speed 1324.88 words/sec, time elapsed 3775.12 sec\n",
            "epoch 36, iter 14180, avg. loss 64.22, avg. ppl 18.60 cum. examples 9276, speed 1565.53 words/sec, time elapsed 3777.37 sec\n",
            "epoch 36, iter 14190, avg. loss 77.08, avg. ppl 23.26 cum. examples 9436, speed 1153.95 words/sec, time elapsed 3780.77 sec\n",
            "epoch 36, iter 14200, avg. loss 57.33, avg. ppl 16.82 cum. examples 9596, speed 1307.22 words/sec, time elapsed 3783.25 sec\n",
            "epoch 36, iter 14210, avg. loss 82.79, avg. ppl 23.75 cum. examples 9756, speed 1322.72 words/sec, time elapsed 3786.41 sec\n",
            "epoch 36, iter 14220, avg. loss 62.04, avg. ppl 19.12 cum. examples 9916, speed 1416.08 words/sec, time elapsed 3788.79 sec\n",
            "epoch 36, iter 14230, avg. loss 68.43, avg. ppl 20.71 cum. examples 10076, speed 1256.06 words/sec, time elapsed 3791.67 sec\n",
            "epoch 36, iter 14240, avg. loss 65.26, avg. ppl 18.39 cum. examples 10236, speed 1391.82 words/sec, time elapsed 3794.24 sec\n",
            "epoch 36, iter 14250, avg. loss 59.20, avg. ppl 17.17 cum. examples 10396, speed 1565.66 words/sec, time elapsed 3796.37 sec\n",
            "epoch 36, iter 14260, avg. loss 69.62, avg. ppl 21.55 cum. examples 10556, speed 1434.43 words/sec, time elapsed 3798.90 sec\n",
            "epoch 36, iter 14270, avg. loss 62.41, avg. ppl 17.47 cum. examples 10716, speed 1190.78 words/sec, time elapsed 3801.83 sec\n",
            "epoch 36, iter 14280, avg. loss 62.91, avg. ppl 18.37 cum. examples 10876, speed 1348.35 words/sec, time elapsed 3804.40 sec\n",
            "epoch 36, iter 14290, avg. loss 65.49, avg. ppl 19.47 cum. examples 11036, speed 1367.53 words/sec, time elapsed 3806.98 sec\n",
            "epoch 36, iter 14300, avg. loss 66.83, avg. ppl 20.68 cum. examples 11196, speed 1316.12 words/sec, time elapsed 3809.66 sec\n",
            "epoch 36, iter 14310, avg. loss 65.05, avg. ppl 18.25 cum. examples 11356, speed 1249.33 words/sec, time elapsed 3812.53 sec\n",
            "epoch 36, iter 14320, avg. loss 65.96, avg. ppl 19.82 cum. examples 11516, speed 1394.85 words/sec, time elapsed 3815.06 sec\n",
            "epoch 36, iter 14330, avg. loss 63.19, avg. ppl 19.92 cum. examples 11676, speed 1157.00 words/sec, time elapsed 3817.98 sec\n",
            "epoch 36, iter 14340, avg. loss 65.50, avg. ppl 20.34 cum. examples 11836, speed 1377.11 words/sec, time elapsed 3820.51 sec\n",
            "epoch 36, iter 14350, avg. loss 60.02, avg. ppl 17.44 cum. examples 11996, speed 1638.61 words/sec, time elapsed 3822.56 sec\n",
            "epoch 36, iter 14360, avg. loss 66.12, avg. ppl 22.07 cum. examples 12156, speed 1439.99 words/sec, time elapsed 3824.93 sec\n",
            "epoch 36, iter 14370, avg. loss 62.81, avg. ppl 18.20 cum. examples 12316, speed 1396.90 words/sec, time elapsed 3827.41 sec\n",
            "epoch 36, iter 14380, avg. loss 60.67, avg. ppl 18.13 cum. examples 12476, speed 1595.67 words/sec, time elapsed 3829.51 sec\n",
            "epoch 36, iter 14390, avg. loss 65.18, avg. ppl 20.21 cum. examples 12636, speed 1369.37 words/sec, time elapsed 3832.05 sec\n",
            "epoch 36, iter 14400, avg. loss 69.80, avg. ppl 20.01 cum. examples 12796, speed 1309.99 words/sec, time elapsed 3834.89 sec\n",
            "epoch 36, iter 14400, cum. loss 68.11, cum. ppl 21.53 cum. examples 12796\n",
            "begin validation ...\n",
            "validation: iter 14400, dev. ppl 64.177234\n",
            "hit patience 2\n",
            "epoch 36, iter 14410, avg. loss 51.98, avg. ppl 14.45 cum. examples 160, speed 400.30 words/sec, time elapsed 3842.67 sec\n",
            "epoch 36, iter 14420, avg. loss 66.04, avg. ppl 19.12 cum. examples 320, speed 1430.98 words/sec, time elapsed 3845.17 sec\n",
            "epoch 36, iter 14430, avg. loss 61.79, avg. ppl 17.89 cum. examples 480, speed 1431.66 words/sec, time elapsed 3847.57 sec\n",
            "epoch 37, iter 14440, avg. loss 68.76, avg. ppl 19.31 cum. examples 638, speed 1446.31 words/sec, time elapsed 3850.10 sec\n",
            "epoch 37, iter 14450, avg. loss 57.20, avg. ppl 14.26 cum. examples 798, speed 1351.34 words/sec, time elapsed 3852.65 sec\n",
            "epoch 37, iter 14460, avg. loss 52.49, avg. ppl 13.40 cum. examples 958, speed 1554.32 words/sec, time elapsed 3854.73 sec\n",
            "epoch 37, iter 14470, avg. loss 60.53, avg. ppl 16.58 cum. examples 1118, speed 1262.82 words/sec, time elapsed 3857.47 sec\n",
            "epoch 37, iter 14480, avg. loss 59.34, avg. ppl 15.90 cum. examples 1278, speed 1416.49 words/sec, time elapsed 3859.89 sec\n",
            "epoch 37, iter 14490, avg. loss 61.81, avg. ppl 17.12 cum. examples 1438, speed 1227.69 words/sec, time elapsed 3862.73 sec\n",
            "epoch 37, iter 14500, avg. loss 73.91, avg. ppl 18.70 cum. examples 1598, speed 1403.80 words/sec, time elapsed 3865.60 sec\n",
            "epoch 37, iter 14510, avg. loss 57.01, avg. ppl 13.88 cum. examples 1758, speed 1386.36 words/sec, time elapsed 3868.10 sec\n",
            "epoch 37, iter 14520, avg. loss 67.74, avg. ppl 17.50 cum. examples 1918, speed 986.85 words/sec, time elapsed 3871.94 sec\n",
            "epoch 37, iter 14530, avg. loss 67.10, avg. ppl 17.85 cum. examples 2078, speed 1234.67 words/sec, time elapsed 3874.96 sec\n",
            "epoch 37, iter 14540, avg. loss 62.33, avg. ppl 16.16 cum. examples 2238, speed 1089.92 words/sec, time elapsed 3878.25 sec\n",
            "epoch 37, iter 14550, avg. loss 57.32, avg. ppl 14.16 cum. examples 2398, speed 1304.05 words/sec, time elapsed 3880.90 sec\n",
            "epoch 37, iter 14560, avg. loss 59.55, avg. ppl 16.83 cum. examples 2558, speed 1300.24 words/sec, time elapsed 3883.49 sec\n",
            "epoch 37, iter 14570, avg. loss 67.85, avg. ppl 17.02 cum. examples 2718, speed 1303.01 words/sec, time elapsed 3886.43 sec\n",
            "epoch 37, iter 14580, avg. loss 51.58, avg. ppl 13.45 cum. examples 2878, speed 1450.71 words/sec, time elapsed 3888.62 sec\n",
            "epoch 37, iter 14590, avg. loss 69.76, avg. ppl 17.92 cum. examples 3038, speed 1272.04 words/sec, time elapsed 3891.66 sec\n",
            "epoch 37, iter 14600, avg. loss 59.79, avg. ppl 14.32 cum. examples 3198, speed 1423.25 words/sec, time elapsed 3894.19 sec\n",
            "epoch 37, iter 14610, avg. loss 56.71, avg. ppl 15.44 cum. examples 3358, speed 1626.79 words/sec, time elapsed 3896.23 sec\n",
            "epoch 37, iter 14620, avg. loss 63.53, avg. ppl 16.02 cum. examples 3518, speed 1440.96 words/sec, time elapsed 3898.77 sec\n",
            "epoch 37, iter 14630, avg. loss 55.90, avg. ppl 15.00 cum. examples 3678, speed 1629.77 words/sec, time elapsed 3900.80 sec\n",
            "epoch 37, iter 14640, avg. loss 53.86, avg. ppl 13.61 cum. examples 3838, speed 1408.25 words/sec, time elapsed 3903.14 sec\n",
            "epoch 37, iter 14650, avg. loss 58.61, avg. ppl 15.99 cum. examples 3998, speed 1457.74 words/sec, time elapsed 3905.46 sec\n",
            "epoch 37, iter 14660, avg. loss 65.17, avg. ppl 17.22 cum. examples 4158, speed 1213.45 words/sec, time elapsed 3908.48 sec\n",
            "epoch 37, iter 14670, avg. loss 66.64, avg. ppl 16.02 cum. examples 4318, speed 1474.64 words/sec, time elapsed 3911.09 sec\n",
            "epoch 37, iter 14680, avg. loss 67.71, avg. ppl 17.89 cum. examples 4478, speed 1470.87 words/sec, time elapsed 3913.64 sec\n",
            "epoch 37, iter 14690, avg. loss 57.31, avg. ppl 15.46 cum. examples 4638, speed 1396.68 words/sec, time elapsed 3916.04 sec\n",
            "epoch 37, iter 14700, avg. loss 61.95, avg. ppl 16.54 cum. examples 4798, speed 1338.81 words/sec, time elapsed 3918.68 sec\n",
            "epoch 37, iter 14710, avg. loss 74.48, avg. ppl 19.37 cum. examples 4958, speed 1250.50 words/sec, time elapsed 3921.89 sec\n",
            "epoch 37, iter 14720, avg. loss 55.02, avg. ppl 14.38 cum. examples 5118, speed 1424.18 words/sec, time elapsed 3924.21 sec\n",
            "epoch 37, iter 14730, avg. loss 55.88, avg. ppl 14.76 cum. examples 5278, speed 1458.03 words/sec, time elapsed 3926.49 sec\n",
            "epoch 37, iter 14740, avg. loss 62.58, avg. ppl 15.88 cum. examples 5438, speed 1371.64 words/sec, time elapsed 3929.13 sec\n",
            "epoch 37, iter 14750, avg. loss 60.12, avg. ppl 15.77 cum. examples 5598, speed 1398.01 words/sec, time elapsed 3931.63 sec\n",
            "epoch 37, iter 14760, avg. loss 63.43, avg. ppl 16.94 cum. examples 5758, speed 1246.76 words/sec, time elapsed 3934.50 sec\n",
            "epoch 37, iter 14770, avg. loss 57.45, avg. ppl 15.11 cum. examples 5918, speed 1252.34 words/sec, time elapsed 3937.21 sec\n",
            "epoch 37, iter 14780, avg. loss 56.49, avg. ppl 14.58 cum. examples 6078, speed 1589.52 words/sec, time elapsed 3939.33 sec\n",
            "epoch 37, iter 14790, avg. loss 58.77, avg. ppl 15.30 cum. examples 6238, speed 1401.32 words/sec, time elapsed 3941.79 sec\n",
            "epoch 37, iter 14800, avg. loss 68.68, avg. ppl 17.99 cum. examples 6398, speed 1584.71 words/sec, time elapsed 3944.19 sec\n",
            "epoch 37, iter 14810, avg. loss 63.28, avg. ppl 16.74 cum. examples 6558, speed 1418.89 words/sec, time elapsed 3946.72 sec\n",
            "epoch 37, iter 14820, avg. loss 60.42, avg. ppl 15.77 cum. examples 6718, speed 1275.68 words/sec, time elapsed 3949.47 sec\n",
            "epoch 37, iter 14830, avg. loss 58.81, avg. ppl 15.15 cum. examples 6878, speed 1477.27 words/sec, time elapsed 3951.81 sec\n",
            "epoch 38, iter 14840, avg. loss 56.65, avg. ppl 15.68 cum. examples 7036, speed 1241.44 words/sec, time elapsed 3954.43 sec\n",
            "epoch 38, iter 14850, avg. loss 55.74, avg. ppl 13.56 cum. examples 7196, speed 1343.96 words/sec, time elapsed 3956.98 sec\n",
            "epoch 38, iter 14860, avg. loss 56.55, avg. ppl 12.60 cum. examples 7356, speed 1288.17 words/sec, time elapsed 3959.75 sec\n",
            "epoch 38, iter 14870, avg. loss 57.86, avg. ppl 12.77 cum. examples 7516, speed 1349.24 words/sec, time elapsed 3962.44 sec\n",
            "epoch 38, iter 14880, avg. loss 56.36, avg. ppl 13.28 cum. examples 7676, speed 1311.05 words/sec, time elapsed 3965.10 sec\n",
            "epoch 38, iter 14890, avg. loss 56.90, avg. ppl 12.78 cum. examples 7836, speed 1282.11 words/sec, time elapsed 3967.89 sec\n",
            "epoch 38, iter 14900, avg. loss 47.95, avg. ppl 10.85 cum. examples 7996, speed 1536.69 words/sec, time elapsed 3969.98 sec\n",
            "epoch 38, iter 14910, avg. loss 63.00, avg. ppl 14.67 cum. examples 8156, speed 1264.40 words/sec, time elapsed 3972.95 sec\n",
            "epoch 38, iter 14920, avg. loss 54.37, avg. ppl 12.48 cum. examples 8316, speed 1342.08 words/sec, time elapsed 3975.52 sec\n",
            "epoch 38, iter 14930, avg. loss 53.12, avg. ppl 12.27 cum. examples 8476, speed 1474.01 words/sec, time elapsed 3977.82 sec\n",
            "epoch 38, iter 14940, avg. loss 50.55, avg. ppl 12.19 cum. examples 8636, speed 1185.53 words/sec, time elapsed 3980.55 sec\n",
            "epoch 38, iter 14950, avg. loss 57.05, avg. ppl 12.91 cum. examples 8796, speed 1407.49 words/sec, time elapsed 3983.08 sec\n",
            "epoch 38, iter 14960, avg. loss 66.12, avg. ppl 15.44 cum. examples 8956, speed 1426.90 words/sec, time elapsed 3985.79 sec\n",
            "epoch 38, iter 14970, avg. loss 63.57, avg. ppl 14.12 cum. examples 9116, speed 1482.05 words/sec, time elapsed 3988.38 sec\n",
            "epoch 38, iter 14980, avg. loss 67.89, avg. ppl 15.50 cum. examples 9276, speed 1447.33 words/sec, time elapsed 3991.12 sec\n",
            "epoch 38, iter 14990, avg. loss 60.03, avg. ppl 13.54 cum. examples 9436, speed 1284.15 words/sec, time elapsed 3993.99 sec\n",
            "epoch 38, iter 15000, avg. loss 51.97, avg. ppl 11.74 cum. examples 9596, speed 1499.30 words/sec, time elapsed 3996.24 sec\n",
            "epoch 38, iter 15010, avg. loss 54.33, avg. ppl 13.05 cum. examples 9756, speed 1555.81 words/sec, time elapsed 3998.42 sec\n",
            "epoch 38, iter 15020, avg. loss 48.82, avg. ppl 10.76 cum. examples 9916, speed 1410.84 words/sec, time elapsed 4000.75 sec\n",
            "epoch 38, iter 15030, avg. loss 60.44, avg. ppl 14.32 cum. examples 10076, speed 1508.45 words/sec, time elapsed 4003.16 sec\n",
            "epoch 38, iter 15040, avg. loss 56.66, avg. ppl 13.42 cum. examples 10236, speed 1393.43 words/sec, time elapsed 4005.66 sec\n",
            "epoch 38, iter 15050, avg. loss 67.10, avg. ppl 15.53 cum. examples 10396, speed 1306.40 words/sec, time elapsed 4008.66 sec\n",
            "epoch 38, iter 15060, avg. loss 53.41, avg. ppl 12.59 cum. examples 10556, speed 1518.74 words/sec, time elapsed 4010.88 sec\n",
            "epoch 38, iter 15070, avg. loss 58.66, avg. ppl 13.56 cum. examples 10716, speed 1272.89 words/sec, time elapsed 4013.71 sec\n",
            "epoch 38, iter 15080, avg. loss 55.89, avg. ppl 12.40 cum. examples 10876, speed 1564.32 words/sec, time elapsed 4015.98 sec\n",
            "epoch 38, iter 15090, avg. loss 51.20, avg. ppl 11.65 cum. examples 11036, speed 1140.07 words/sec, time elapsed 4018.91 sec\n",
            "epoch 38, iter 15100, avg. loss 57.41, avg. ppl 14.55 cum. examples 11196, speed 1265.45 words/sec, time elapsed 4021.62 sec\n",
            "epoch 38, iter 15110, avg. loss 62.83, avg. ppl 14.12 cum. examples 11356, speed 1452.78 words/sec, time elapsed 4024.23 sec\n",
            "epoch 38, iter 15120, avg. loss 54.27, avg. ppl 13.32 cum. examples 11516, speed 1680.70 words/sec, time elapsed 4026.23 sec\n",
            "epoch 38, iter 15130, avg. loss 65.50, avg. ppl 15.96 cum. examples 11676, speed 1476.13 words/sec, time elapsed 4028.79 sec\n",
            "epoch 38, iter 15140, avg. loss 56.35, avg. ppl 13.34 cum. examples 11836, speed 1269.38 words/sec, time elapsed 4031.53 sec\n",
            "epoch 38, iter 15150, avg. loss 60.51, avg. ppl 14.44 cum. examples 11996, speed 1431.23 words/sec, time elapsed 4034.06 sec\n",
            "epoch 38, iter 15160, avg. loss 56.89, avg. ppl 13.32 cum. examples 12156, speed 1497.75 words/sec, time elapsed 4036.41 sec\n",
            "epoch 38, iter 15170, avg. loss 53.68, avg. ppl 12.83 cum. examples 12316, speed 1258.88 words/sec, time elapsed 4039.09 sec\n",
            "epoch 38, iter 15180, avg. loss 53.09, avg. ppl 12.86 cum. examples 12476, speed 1177.48 words/sec, time elapsed 4041.91 sec\n",
            "epoch 38, iter 15190, avg. loss 56.04, avg. ppl 12.74 cum. examples 12636, speed 1298.75 words/sec, time elapsed 4044.62 sec\n",
            "epoch 38, iter 15200, avg. loss 57.15, avg. ppl 13.87 cum. examples 12796, speed 1014.36 words/sec, time elapsed 4048.05 sec\n",
            "epoch 38, iter 15200, cum. loss 59.41, cum. ppl 14.77 cum. examples 12796\n",
            "begin validation ...\n",
            "validation: iter 15200, dev. ppl 76.303897\n",
            "hit patience 3\n",
            "epoch 38, iter 15210, avg. loss 46.14, avg. ppl 11.06 cum. examples 160, speed 421.47 words/sec, time elapsed 4055.34 sec\n",
            "epoch 38, iter 15220, avg. loss 57.89, avg. ppl 14.77 cum. examples 320, speed 1264.08 words/sec, time elapsed 4058.06 sec\n",
            "epoch 38, iter 15230, avg. loss 64.10, avg. ppl 15.73 cum. examples 480, speed 1292.47 words/sec, time elapsed 4060.94 sec\n",
            "epoch 39, iter 15240, avg. loss 77.18, avg. ppl 17.87 cum. examples 638, speed 1167.68 words/sec, time elapsed 4064.56 sec\n",
            "epoch 39, iter 15250, avg. loss 55.40, avg. ppl 11.88 cum. examples 798, speed 1405.91 words/sec, time elapsed 4067.11 sec\n",
            "epoch 39, iter 15260, avg. loss 44.95, avg. ppl 9.22 cum. examples 958, speed 1299.53 words/sec, time elapsed 4069.60 sec\n",
            "epoch 39, iter 15270, avg. loss 49.26, avg. ppl 10.12 cum. examples 1118, speed 1409.40 words/sec, time elapsed 4072.02 sec\n",
            "epoch 39, iter 15280, avg. loss 61.04, avg. ppl 12.80 cum. examples 1278, speed 1284.63 words/sec, time elapsed 4075.00 sec\n",
            "epoch 39, iter 15290, avg. loss 51.46, avg. ppl 10.41 cum. examples 1438, speed 1698.00 words/sec, time elapsed 4077.07 sec\n",
            "epoch 39, iter 15300, avg. loss 59.07, avg. ppl 12.26 cum. examples 1598, speed 1411.32 words/sec, time elapsed 4079.74 sec\n",
            "epoch 39, iter 15310, avg. loss 53.23, avg. ppl 11.08 cum. examples 1758, speed 1629.21 words/sec, time elapsed 4081.91 sec\n",
            "epoch 39, iter 15320, avg. loss 54.10, avg. ppl 10.88 cum. examples 1918, speed 1271.22 words/sec, time elapsed 4084.77 sec\n",
            "epoch 39, iter 15330, avg. loss 48.49, avg. ppl 9.80 cum. examples 2078, speed 1509.90 words/sec, time elapsed 4087.02 sec\n",
            "epoch 39, iter 15340, avg. loss 53.33, avg. ppl 10.97 cum. examples 2238, speed 1686.45 words/sec, time elapsed 4089.13 sec\n",
            "epoch 39, iter 15350, avg. loss 53.84, avg. ppl 11.98 cum. examples 2398, speed 1180.46 words/sec, time elapsed 4092.07 sec\n",
            "epoch 39, iter 15360, avg. loss 63.70, avg. ppl 13.45 cum. examples 2558, speed 1218.30 words/sec, time elapsed 4095.29 sec\n",
            "epoch 39, iter 15370, avg. loss 50.99, avg. ppl 10.61 cum. examples 2718, speed 1297.65 words/sec, time elapsed 4097.95 sec\n",
            "epoch 39, iter 15380, avg. loss 52.13, avg. ppl 11.13 cum. examples 2878, speed 1609.64 words/sec, time elapsed 4100.10 sec\n",
            "epoch 39, iter 15390, avg. loss 50.34, avg. ppl 10.43 cum. examples 3038, speed 1479.16 words/sec, time elapsed 4102.42 sec\n",
            "epoch 39, iter 15400, avg. loss 55.21, avg. ppl 11.91 cum. examples 3198, speed 1410.97 words/sec, time elapsed 4104.95 sec\n",
            "epoch 39, iter 15410, avg. loss 52.12, avg. ppl 10.48 cum. examples 3358, speed 1198.62 words/sec, time elapsed 4107.91 sec\n",
            "epoch 39, iter 15420, avg. loss 62.89, avg. ppl 13.55 cum. examples 3518, speed 1147.33 words/sec, time elapsed 4111.28 sec\n",
            "epoch 39, iter 15430, avg. loss 44.66, avg. ppl 9.00 cum. examples 3678, speed 1584.24 words/sec, time elapsed 4113.33 sec\n",
            "epoch 39, iter 15440, avg. loss 44.70, avg. ppl 9.73 cum. examples 3838, speed 1378.00 words/sec, time elapsed 4115.61 sec\n",
            "epoch 39, iter 15450, avg. loss 50.44, avg. ppl 10.76 cum. examples 3998, speed 1405.75 words/sec, time elapsed 4118.03 sec\n",
            "epoch 39, iter 15460, avg. loss 55.02, avg. ppl 12.01 cum. examples 4158, speed 1255.67 words/sec, time elapsed 4120.85 sec\n",
            "epoch 39, iter 15470, avg. loss 51.93, avg. ppl 11.05 cum. examples 4318, speed 1279.99 words/sec, time elapsed 4123.55 sec\n",
            "epoch 39, iter 15480, avg. loss 49.33, avg. ppl 10.59 cum. examples 4478, speed 1364.19 words/sec, time elapsed 4126.00 sec\n",
            "epoch 39, iter 15490, avg. loss 50.72, avg. ppl 10.73 cum. examples 4638, speed 1381.04 words/sec, time elapsed 4128.48 sec\n",
            "epoch 39, iter 15500, avg. loss 53.84, avg. ppl 11.05 cum. examples 4798, speed 1584.79 words/sec, time elapsed 4130.74 sec\n",
            "epoch 39, iter 15510, avg. loss 50.26, avg. ppl 10.46 cum. examples 4958, speed 1604.45 words/sec, time elapsed 4132.87 sec\n",
            "epoch 39, iter 15520, avg. loss 54.97, avg. ppl 11.77 cum. examples 5118, speed 1477.95 words/sec, time elapsed 4135.29 sec\n",
            "epoch 39, iter 15530, avg. loss 65.80, avg. ppl 14.69 cum. examples 5278, speed 1467.27 words/sec, time elapsed 4137.96 sec\n",
            "epoch 39, iter 15540, avg. loss 51.82, avg. ppl 11.18 cum. examples 5438, speed 1406.67 words/sec, time elapsed 4140.40 sec\n",
            "epoch 39, iter 15550, avg. loss 48.34, avg. ppl 10.71 cum. examples 5598, speed 1503.48 words/sec, time elapsed 4142.57 sec\n",
            "epoch 39, iter 15560, avg. loss 61.00, avg. ppl 14.14 cum. examples 5758, speed 1200.79 words/sec, time elapsed 4145.64 sec\n",
            "epoch 39, iter 15570, avg. loss 53.68, avg. ppl 12.40 cum. examples 5918, speed 1225.96 words/sec, time elapsed 4148.42 sec\n",
            "epoch 39, iter 15580, avg. loss 57.50, avg. ppl 12.02 cum. examples 6078, speed 1402.40 words/sec, time elapsed 4151.06 sec\n",
            "epoch 39, iter 15590, avg. loss 56.73, avg. ppl 11.31 cum. examples 6238, speed 1392.51 words/sec, time elapsed 4153.74 sec\n",
            "epoch 39, iter 15600, avg. loss 54.30, avg. ppl 11.57 cum. examples 6398, speed 1468.46 words/sec, time elapsed 4156.16 sec\n",
            "epoch 39, iter 15610, avg. loss 60.48, avg. ppl 12.15 cum. examples 6558, speed 1398.72 words/sec, time elapsed 4158.93 sec\n",
            "epoch 39, iter 15620, avg. loss 62.89, avg. ppl 13.75 cum. examples 6718, speed 1494.70 words/sec, time elapsed 4161.50 sec\n",
            "epoch 39, iter 15630, avg. loss 44.12, avg. ppl 9.33 cum. examples 6878, speed 1302.69 words/sec, time elapsed 4163.93 sec\n",
            "epoch 40, iter 15640, avg. loss 51.04, avg. ppl 10.89 cum. examples 7036, speed 1286.48 words/sec, time elapsed 4166.55 sec\n",
            "epoch 40, iter 15650, avg. loss 48.44, avg. ppl 8.78 cum. examples 7196, speed 1386.04 words/sec, time elapsed 4169.13 sec\n",
            "epoch 40, iter 15660, avg. loss 51.62, avg. ppl 9.82 cum. examples 7356, speed 1626.08 words/sec, time elapsed 4171.35 sec\n",
            "epoch 40, iter 15670, avg. loss 43.80, avg. ppl 8.31 cum. examples 7516, speed 1636.72 words/sec, time elapsed 4173.37 sec\n",
            "epoch 40, iter 15680, avg. loss 42.82, avg. ppl 8.37 cum. examples 7676, speed 1478.40 words/sec, time elapsed 4175.55 sec\n",
            "epoch 40, iter 15690, avg. loss 52.14, avg. ppl 10.40 cum. examples 7836, speed 1423.71 words/sec, time elapsed 4178.05 sec\n",
            "epoch 40, iter 15700, avg. loss 50.48, avg. ppl 9.76 cum. examples 7996, speed 1329.38 words/sec, time elapsed 4180.72 sec\n",
            "epoch 40, iter 15710, avg. loss 52.20, avg. ppl 9.71 cum. examples 8156, speed 1502.12 words/sec, time elapsed 4183.17 sec\n",
            "epoch 40, iter 15720, avg. loss 45.80, avg. ppl 8.59 cum. examples 8316, speed 1300.07 words/sec, time elapsed 4185.79 sec\n",
            "epoch 40, iter 15730, avg. loss 46.65, avg. ppl 8.96 cum. examples 8476, speed 1170.27 words/sec, time elapsed 4188.70 sec\n",
            "epoch 40, iter 15740, avg. loss 48.10, avg. ppl 8.81 cum. examples 8636, speed 1406.59 words/sec, time elapsed 4191.21 sec\n",
            "epoch 40, iter 15750, avg. loss 53.23, avg. ppl 10.25 cum. examples 8796, speed 1382.60 words/sec, time elapsed 4193.86 sec\n",
            "epoch 40, iter 15760, avg. loss 54.90, avg. ppl 10.44 cum. examples 8956, speed 1345.45 words/sec, time elapsed 4196.64 sec\n",
            "epoch 40, iter 15770, avg. loss 51.78, avg. ppl 10.00 cum. examples 9116, speed 1444.47 words/sec, time elapsed 4199.13 sec\n",
            "epoch 40, iter 15780, avg. loss 50.89, avg. ppl 9.83 cum. examples 9276, speed 1122.72 words/sec, time elapsed 4202.31 sec\n",
            "epoch 40, iter 15790, avg. loss 41.77, avg. ppl 8.32 cum. examples 9436, speed 1416.30 words/sec, time elapsed 4204.53 sec\n",
            "epoch 40, iter 15800, avg. loss 49.88, avg. ppl 9.60 cum. examples 9596, speed 1240.19 words/sec, time elapsed 4207.38 sec\n",
            "epoch 40, iter 15810, avg. loss 50.38, avg. ppl 9.75 cum. examples 9756, speed 1355.60 words/sec, time elapsed 4209.99 sec\n",
            "epoch 40, iter 15820, avg. loss 60.77, avg. ppl 11.45 cum. examples 9916, speed 1198.02 words/sec, time elapsed 4213.32 sec\n",
            "epoch 40, iter 15830, avg. loss 50.56, avg. ppl 9.39 cum. examples 10076, speed 1560.36 words/sec, time elapsed 4215.63 sec\n",
            "epoch 40, iter 15840, avg. loss 55.51, avg. ppl 10.57 cum. examples 10236, speed 1392.40 words/sec, time elapsed 4218.34 sec\n",
            "epoch 40, iter 15850, avg. loss 44.46, avg. ppl 8.22 cum. examples 10396, speed 1541.06 words/sec, time elapsed 4220.53 sec\n",
            "epoch 40, iter 15860, avg. loss 47.00, avg. ppl 8.83 cum. examples 10556, speed 1533.43 words/sec, time elapsed 4222.78 sec\n",
            "epoch 40, iter 15870, avg. loss 58.20, avg. ppl 11.47 cum. examples 10716, speed 1151.44 words/sec, time elapsed 4226.10 sec\n",
            "epoch 40, iter 15880, avg. loss 47.53, avg. ppl 9.54 cum. examples 10876, speed 1492.07 words/sec, time elapsed 4228.36 sec\n",
            "epoch 40, iter 15890, avg. loss 57.54, avg. ppl 11.20 cum. examples 11036, speed 1526.14 words/sec, time elapsed 4230.85 sec\n",
            "epoch 40, iter 15900, avg. loss 49.64, avg. ppl 9.45 cum. examples 11196, speed 1559.77 words/sec, time elapsed 4233.12 sec\n",
            "epoch 40, iter 15910, avg. loss 51.65, avg. ppl 10.42 cum. examples 11356, speed 1099.91 words/sec, time elapsed 4236.32 sec\n",
            "epoch 40, iter 15920, avg. loss 49.28, avg. ppl 9.63 cum. examples 11516, speed 1387.94 words/sec, time elapsed 4238.83 sec\n",
            "epoch 40, iter 15930, avg. loss 56.79, avg. ppl 11.54 cum. examples 11676, speed 1417.36 words/sec, time elapsed 4241.45 sec\n",
            "epoch 40, iter 15940, avg. loss 47.75, avg. ppl 9.74 cum. examples 11836, speed 1320.41 words/sec, time elapsed 4244.00 sec\n",
            "epoch 40, iter 15950, avg. loss 55.83, avg. ppl 10.36 cum. examples 11996, speed 1504.73 words/sec, time elapsed 4246.53 sec\n",
            "epoch 40, iter 15960, avg. loss 41.35, avg. ppl 8.10 cum. examples 12156, speed 1495.79 words/sec, time elapsed 4248.65 sec\n",
            "epoch 40, iter 15970, avg. loss 47.70, avg. ppl 9.26 cum. examples 12316, speed 1596.89 words/sec, time elapsed 4250.80 sec\n",
            "epoch 40, iter 15980, avg. loss 49.83, avg. ppl 10.04 cum. examples 12476, speed 1169.13 words/sec, time elapsed 4253.75 sec\n",
            "epoch 40, iter 15990, avg. loss 60.80, avg. ppl 12.12 cum. examples 12636, speed 1405.69 words/sec, time elapsed 4256.53 sec\n",
            "epoch 40, iter 16000, avg. loss 54.39, avg. ppl 10.11 cum. examples 12796, speed 1288.29 words/sec, time elapsed 4259.45 sec\n",
            "epoch 40, iter 16000, cum. loss 52.64, cum. ppl 10.74 cum. examples 12796\n",
            "begin validation ...\n",
            "validation: iter 16000, dev. ppl 90.208399\n",
            "hit patience 4\n",
            "epoch 40, iter 16010, avg. loss 40.78, avg. ppl 8.09 cum. examples 160, speed 422.82 words/sec, time elapsed 4266.83 sec\n",
            "epoch 40, iter 16020, avg. loss 49.13, avg. ppl 9.53 cum. examples 320, speed 1171.38 words/sec, time elapsed 4269.81 sec\n",
            "epoch 40, iter 16030, avg. loss 49.63, avg. ppl 9.95 cum. examples 480, speed 1172.94 words/sec, time elapsed 4272.75 sec\n",
            "epoch 40, iter 16040, avg. loss 47.78, avg. ppl 9.30 cum. examples 638, speed 1309.50 words/sec, time elapsed 4275.34 sec\n",
            "epoch 41, iter 16050, avg. loss 45.30, avg. ppl 8.42 cum. examples 798, speed 1318.43 words/sec, time elapsed 4277.92 sec\n",
            "epoch 41, iter 16060, avg. loss 43.31, avg. ppl 7.52 cum. examples 958, speed 1315.46 words/sec, time elapsed 4280.53 sec\n",
            "epoch 41, iter 16070, avg. loss 40.28, avg. ppl 7.14 cum. examples 1118, speed 1355.39 words/sec, time elapsed 4282.95 sec\n",
            "epoch 41, iter 16080, avg. loss 52.91, avg. ppl 9.41 cum. examples 1278, speed 1336.48 words/sec, time elapsed 4285.77 sec\n",
            "epoch 41, iter 16090, avg. loss 45.62, avg. ppl 8.03 cum. examples 1438, speed 1265.27 words/sec, time elapsed 4288.54 sec\n",
            "epoch 41, iter 16100, avg. loss 39.51, avg. ppl 6.67 cum. examples 1598, speed 1697.84 words/sec, time elapsed 4290.51 sec\n",
            "epoch 41, iter 16110, avg. loss 51.14, avg. ppl 8.35 cum. examples 1758, speed 1410.72 words/sec, time elapsed 4293.24 sec\n",
            "epoch 41, iter 16120, avg. loss 52.45, avg. ppl 8.99 cum. examples 1918, speed 1451.68 words/sec, time elapsed 4295.87 sec\n",
            "epoch 41, iter 16130, avg. loss 47.08, avg. ppl 8.48 cum. examples 2078, speed 1488.81 words/sec, time elapsed 4298.24 sec\n",
            "epoch 41, iter 16140, avg. loss 42.04, avg. ppl 7.30 cum. examples 2238, speed 1553.34 words/sec, time elapsed 4300.42 sec\n",
            "epoch 41, iter 16150, avg. loss 47.64, avg. ppl 8.70 cum. examples 2398, speed 1305.43 words/sec, time elapsed 4303.11 sec\n",
            "epoch 41, iter 16160, avg. loss 40.40, avg. ppl 7.23 cum. examples 2558, speed 1538.89 words/sec, time elapsed 4305.24 sec\n",
            "epoch 41, iter 16170, avg. loss 50.49, avg. ppl 8.76 cum. examples 2718, speed 1271.62 words/sec, time elapsed 4308.16 sec\n",
            "epoch 41, iter 16180, avg. loss 46.75, avg. ppl 8.53 cum. examples 2878, speed 1321.68 words/sec, time elapsed 4310.80 sec\n",
            "epoch 41, iter 16190, avg. loss 39.93, avg. ppl 7.03 cum. examples 3038, speed 1531.34 words/sec, time elapsed 4312.94 sec\n",
            "epoch 41, iter 16200, avg. loss 40.46, avg. ppl 7.21 cum. examples 3198, speed 1359.86 words/sec, time elapsed 4315.35 sec\n",
            "epoch 41, iter 16210, avg. loss 46.33, avg. ppl 7.83 cum. examples 3358, speed 1517.83 words/sec, time elapsed 4317.73 sec\n",
            "epoch 41, iter 16220, avg. loss 46.46, avg. ppl 8.46 cum. examples 3518, speed 1422.48 words/sec, time elapsed 4320.17 sec\n",
            "epoch 41, iter 16230, avg. loss 54.78, avg. ppl 9.96 cum. examples 3678, speed 1188.18 words/sec, time elapsed 4323.38 sec\n",
            "epoch 41, iter 16240, avg. loss 49.60, avg. ppl 8.74 cum. examples 3838, speed 1500.40 words/sec, time elapsed 4325.82 sec\n",
            "epoch 41, iter 16250, avg. loss 50.80, avg. ppl 9.18 cum. examples 3998, speed 1299.35 words/sec, time elapsed 4328.64 sec\n",
            "epoch 41, iter 16260, avg. loss 50.15, avg. ppl 9.22 cum. examples 4158, speed 1334.79 words/sec, time elapsed 4331.35 sec\n",
            "epoch 41, iter 16270, avg. loss 40.59, avg. ppl 7.55 cum. examples 4318, speed 1434.23 words/sec, time elapsed 4333.59 sec\n",
            "epoch 41, iter 16280, avg. loss 43.88, avg. ppl 7.91 cum. examples 4478, speed 1483.83 words/sec, time elapsed 4335.88 sec\n",
            "epoch 41, iter 16290, avg. loss 52.16, avg. ppl 9.20 cum. examples 4638, speed 1334.67 words/sec, time elapsed 4338.69 sec\n",
            "epoch 41, iter 16300, avg. loss 51.77, avg. ppl 8.98 cum. examples 4798, speed 1267.86 words/sec, time elapsed 4341.67 sec\n",
            "epoch 41, iter 16310, avg. loss 46.18, avg. ppl 8.03 cum. examples 4958, speed 1268.54 words/sec, time elapsed 4344.47 sec\n",
            "epoch 41, iter 16320, avg. loss 41.21, avg. ppl 7.21 cum. examples 5118, speed 1281.27 words/sec, time elapsed 4347.07 sec\n",
            "epoch 41, iter 16330, avg. loss 54.33, avg. ppl 9.58 cum. examples 5278, speed 1360.06 words/sec, time elapsed 4349.90 sec\n",
            "epoch 41, iter 16340, avg. loss 44.72, avg. ppl 7.89 cum. examples 5438, speed 1504.64 words/sec, time elapsed 4352.20 sec\n",
            "epoch 41, iter 16350, avg. loss 41.46, avg. ppl 7.44 cum. examples 5598, speed 1400.25 words/sec, time elapsed 4354.56 sec\n",
            "epoch 41, iter 16360, avg. loss 48.95, avg. ppl 8.60 cum. examples 5758, speed 1433.14 words/sec, time elapsed 4357.10 sec\n",
            "epoch 41, iter 16370, avg. loss 48.03, avg. ppl 9.24 cum. examples 5918, speed 1351.88 words/sec, time elapsed 4359.66 sec\n",
            "epoch 41, iter 16380, avg. loss 44.96, avg. ppl 7.93 cum. examples 6078, speed 1372.68 words/sec, time elapsed 4362.19 sec\n",
            "epoch 41, iter 16390, avg. loss 43.85, avg. ppl 7.45 cum. examples 6238, speed 1311.29 words/sec, time elapsed 4364.86 sec\n",
            "epoch 41, iter 16400, avg. loss 42.60, avg. ppl 7.55 cum. examples 6398, speed 1356.70 words/sec, time elapsed 4367.34 sec\n",
            "epoch 41, iter 16410, avg. loss 51.93, avg. ppl 9.63 cum. examples 6558, speed 1355.35 words/sec, time elapsed 4370.05 sec\n",
            "epoch 41, iter 16420, avg. loss 55.29, avg. ppl 10.25 cum. examples 6718, speed 1301.49 words/sec, time elapsed 4372.97 sec\n",
            "epoch 41, iter 16430, avg. loss 48.11, avg. ppl 8.81 cum. examples 6878, speed 1296.20 words/sec, time elapsed 4375.70 sec\n",
            "epoch 41, iter 16440, avg. loss 52.04, avg. ppl 9.34 cum. examples 7038, speed 1266.59 words/sec, time elapsed 4378.64 sec\n",
            "epoch 42, iter 16450, avg. loss 37.40, avg. ppl 5.97 cum. examples 7196, speed 1279.95 words/sec, time elapsed 4381.22 sec\n",
            "epoch 42, iter 16460, avg. loss 32.54, avg. ppl 5.41 cum. examples 7356, speed 1537.50 words/sec, time elapsed 4383.23 sec\n",
            "epoch 42, iter 16470, avg. loss 41.12, avg. ppl 6.84 cum. examples 7516, speed 1495.27 words/sec, time elapsed 4385.52 sec\n",
            "epoch 42, iter 16480, avg. loss 42.30, avg. ppl 6.97 cum. examples 7676, speed 1424.33 words/sec, time elapsed 4387.97 sec\n",
            "epoch 42, iter 16490, avg. loss 49.09, avg. ppl 7.98 cum. examples 7836, speed 1188.49 words/sec, time elapsed 4391.15 sec\n",
            "epoch 42, iter 16500, avg. loss 42.90, avg. ppl 7.08 cum. examples 7996, speed 1380.27 words/sec, time elapsed 4393.69 sec\n",
            "epoch 42, iter 16510, avg. loss 41.18, avg. ppl 6.42 cum. examples 8156, speed 1572.43 words/sec, time elapsed 4395.94 sec\n",
            "epoch 42, iter 16520, avg. loss 48.93, avg. ppl 8.24 cum. examples 8316, speed 1478.32 words/sec, time elapsed 4398.45 sec\n",
            "epoch 42, iter 16530, avg. loss 51.71, avg. ppl 8.40 cum. examples 8476, speed 1351.40 words/sec, time elapsed 4401.33 sec\n",
            "epoch 42, iter 16540, avg. loss 41.72, avg. ppl 7.00 cum. examples 8636, speed 1468.42 words/sec, time elapsed 4403.67 sec\n",
            "epoch 42, iter 16550, avg. loss 36.76, avg. ppl 6.06 cum. examples 8796, speed 1468.52 words/sec, time elapsed 4405.89 sec\n",
            "epoch 42, iter 16560, avg. loss 38.18, avg. ppl 6.41 cum. examples 8956, speed 1453.69 words/sec, time elapsed 4408.15 sec\n",
            "epoch 42, iter 16570, avg. loss 48.33, avg. ppl 8.22 cum. examples 9116, speed 1257.69 words/sec, time elapsed 4411.07 sec\n",
            "epoch 42, iter 16580, avg. loss 47.59, avg. ppl 7.63 cum. examples 9276, speed 1369.01 words/sec, time elapsed 4413.81 sec\n",
            "epoch 42, iter 16590, avg. loss 44.27, avg. ppl 7.27 cum. examples 9436, speed 1514.40 words/sec, time elapsed 4416.16 sec\n",
            "epoch 42, iter 16600, avg. loss 43.34, avg. ppl 7.41 cum. examples 9596, speed 1396.52 words/sec, time elapsed 4418.64 sec\n",
            "epoch 42, iter 16610, avg. loss 56.23, avg. ppl 9.22 cum. examples 9756, speed 1232.53 words/sec, time elapsed 4421.93 sec\n",
            "epoch 42, iter 16620, avg. loss 42.11, avg. ppl 7.07 cum. examples 9916, speed 1352.47 words/sec, time elapsed 4424.48 sec\n",
            "epoch 42, iter 16630, avg. loss 51.66, avg. ppl 8.97 cum. examples 10076, speed 1251.65 words/sec, time elapsed 4427.49 sec\n",
            "epoch 42, iter 16640, avg. loss 44.78, avg. ppl 7.37 cum. examples 10236, speed 1518.00 words/sec, time elapsed 4429.85 sec\n",
            "epoch 42, iter 16650, avg. loss 38.75, avg. ppl 6.91 cum. examples 10396, speed 1352.52 words/sec, time elapsed 4432.22 sec\n",
            "epoch 42, iter 16660, avg. loss 42.15, avg. ppl 6.80 cum. examples 10556, speed 1653.71 words/sec, time elapsed 4434.35 sec\n",
            "epoch 42, iter 16670, avg. loss 55.01, avg. ppl 9.26 cum. examples 10716, speed 1344.70 words/sec, time elapsed 4437.29 sec\n",
            "epoch 42, iter 16680, avg. loss 44.20, avg. ppl 7.17 cum. examples 10876, speed 1217.19 words/sec, time elapsed 4440.24 sec\n",
            "epoch 42, iter 16690, avg. loss 45.82, avg. ppl 7.59 cum. examples 11036, speed 1248.03 words/sec, time elapsed 4443.14 sec\n",
            "epoch 42, iter 16700, avg. loss 39.67, avg. ppl 6.07 cum. examples 11196, speed 1334.75 words/sec, time elapsed 4445.77 sec\n",
            "epoch 42, iter 16710, avg. loss 40.02, avg. ppl 6.57 cum. examples 11356, speed 1413.56 words/sec, time elapsed 4448.18 sec\n",
            "epoch 42, iter 16720, avg. loss 42.18, avg. ppl 6.64 cum. examples 11516, speed 1312.19 words/sec, time elapsed 4450.89 sec\n",
            "epoch 42, iter 16730, avg. loss 29.36, avg. ppl 5.05 cum. examples 11676, speed 1563.51 words/sec, time elapsed 4452.75 sec\n",
            "epoch 42, iter 16740, avg. loss 41.16, avg. ppl 6.91 cum. examples 11836, speed 1567.78 words/sec, time elapsed 4454.92 sec\n",
            "epoch 42, iter 16750, avg. loss 42.45, avg. ppl 7.31 cum. examples 11996, speed 1403.93 words/sec, time elapsed 4457.35 sec\n",
            "epoch 42, iter 16760, avg. loss 50.53, avg. ppl 8.56 cum. examples 12156, speed 1360.76 words/sec, time elapsed 4460.12 sec\n",
            "epoch 42, iter 16770, avg. loss 56.56, avg. ppl 9.47 cum. examples 12316, speed 1360.24 words/sec, time elapsed 4463.08 sec\n",
            "epoch 42, iter 16780, avg. loss 46.80, avg. ppl 7.93 cum. examples 12476, speed 1398.63 words/sec, time elapsed 4465.67 sec\n",
            "epoch 42, iter 16790, avg. loss 38.81, avg. ppl 6.49 cum. examples 12636, speed 1573.97 words/sec, time elapsed 4467.77 sec\n",
            "epoch 42, iter 16800, avg. loss 35.48, avg. ppl 5.65 cum. examples 12796, speed 1604.97 words/sec, time elapsed 4469.82 sec\n",
            "epoch 42, iter 16800, cum. loss 45.42, cum. ppl 7.85 cum. examples 12796\n",
            "begin validation ...\n",
            "validation: iter 16800, dev. ppl 110.950332\n",
            "hit patience 5\n",
            "hit #4 trial\n",
            "load previously best model and decay learning rate to 0.000063\n",
            "restore parameters of the optimizers\n",
            "epoch 42, iter 16810, avg. loss 78.47, avg. ppl 41.96 cum. examples 160, speed 428.19 words/sec, time elapsed 4477.66 sec\n",
            "epoch 42, iter 16820, avg. loss 79.58, avg. ppl 38.65 cum. examples 320, speed 1530.72 words/sec, time elapsed 4479.94 sec\n",
            "epoch 42, iter 16830, avg. loss 83.67, avg. ppl 37.06 cum. examples 480, speed 1098.08 words/sec, time elapsed 4483.32 sec\n",
            "epoch 42, iter 16840, avg. loss 82.14, avg. ppl 37.70 cum. examples 640, speed 1511.00 words/sec, time elapsed 4485.71 sec\n",
            "epoch 43, iter 16850, avg. loss 83.73, avg. ppl 40.06 cum. examples 798, speed 1385.77 words/sec, time elapsed 4488.30 sec\n",
            "epoch 43, iter 16860, avg. loss 101.94, avg. ppl 45.14 cum. examples 958, speed 1212.27 words/sec, time elapsed 4491.83 sec\n",
            "epoch 43, iter 16870, avg. loss 80.54, avg. ppl 38.10 cum. examples 1118, speed 1237.25 words/sec, time elapsed 4494.69 sec\n",
            "epoch 43, iter 16880, avg. loss 76.65, avg. ppl 35.67 cum. examples 1278, speed 1239.84 words/sec, time elapsed 4497.46 sec\n",
            "epoch 43, iter 16890, avg. loss 85.57, avg. ppl 41.11 cum. examples 1438, speed 1412.06 words/sec, time elapsed 4500.07 sec\n",
            "epoch 43, iter 16900, avg. loss 82.06, avg. ppl 36.83 cum. examples 1598, speed 1548.89 words/sec, time elapsed 4502.42 sec\n",
            "epoch 43, iter 16910, avg. loss 73.97, avg. ppl 36.18 cum. examples 1758, speed 1526.39 words/sec, time elapsed 4504.58 sec\n",
            "epoch 43, iter 16920, avg. loss 77.73, avg. ppl 37.87 cum. examples 1918, speed 1265.96 words/sec, time elapsed 4507.28 sec\n",
            "epoch 43, iter 16930, avg. loss 78.75, avg. ppl 37.67 cum. examples 2078, speed 1358.32 words/sec, time elapsed 4509.84 sec\n",
            "epoch 43, iter 16940, avg. loss 83.65, avg. ppl 41.78 cum. examples 2238, speed 1311.01 words/sec, time elapsed 4512.57 sec\n",
            "epoch 43, iter 16950, avg. loss 79.02, avg. ppl 37.47 cum. examples 2398, speed 1127.31 words/sec, time elapsed 4515.67 sec\n",
            "epoch 43, iter 16960, avg. loss 70.07, avg. ppl 31.38 cum. examples 2558, speed 1481.29 words/sec, time elapsed 4517.87 sec\n",
            "epoch 43, iter 16970, avg. loss 84.02, avg. ppl 37.13 cum. examples 2718, speed 1601.16 words/sec, time elapsed 4520.19 sec\n",
            "epoch 43, iter 16980, avg. loss 84.53, avg. ppl 36.35 cum. examples 2878, speed 1217.84 words/sec, time elapsed 4523.28 sec\n",
            "epoch 43, iter 16990, avg. loss 71.76, avg. ppl 35.92 cum. examples 3038, speed 1354.45 words/sec, time elapsed 4525.65 sec\n",
            "epoch 43, iter 17000, avg. loss 64.47, avg. ppl 29.50 cum. examples 3198, speed 1440.10 words/sec, time elapsed 4527.76 sec\n",
            "epoch 43, iter 17010, avg. loss 76.13, avg. ppl 34.19 cum. examples 3358, speed 1384.00 words/sec, time elapsed 4530.25 sec\n",
            "epoch 43, iter 17020, avg. loss 78.65, avg. ppl 36.58 cum. examples 3518, speed 1243.58 words/sec, time elapsed 4533.07 sec\n",
            "epoch 43, iter 17030, avg. loss 75.65, avg. ppl 33.53 cum. examples 3678, speed 1418.70 words/sec, time elapsed 4535.49 sec\n",
            "epoch 43, iter 17040, avg. loss 76.98, avg. ppl 32.96 cum. examples 3838, speed 1329.68 words/sec, time elapsed 4538.15 sec\n",
            "epoch 43, iter 17050, avg. loss 81.93, avg. ppl 36.93 cum. examples 3998, speed 1494.26 words/sec, time elapsed 4540.58 sec\n",
            "epoch 43, iter 17060, avg. loss 77.57, avg. ppl 36.31 cum. examples 4158, speed 1575.54 words/sec, time elapsed 4542.77 sec\n",
            "epoch 43, iter 17070, avg. loss 84.91, avg. ppl 37.63 cum. examples 4318, speed 1256.11 words/sec, time elapsed 4545.75 sec\n",
            "epoch 43, iter 17080, avg. loss 73.98, avg. ppl 33.85 cum. examples 4478, speed 1257.82 words/sec, time elapsed 4548.42 sec\n",
            "epoch 43, iter 17090, avg. loss 72.78, avg. ppl 34.67 cum. examples 4638, speed 1433.07 words/sec, time elapsed 4550.71 sec\n",
            "epoch 43, iter 17100, avg. loss 79.50, avg. ppl 36.83 cum. examples 4798, speed 1458.71 words/sec, time elapsed 4553.13 sec\n",
            "epoch 43, iter 17110, avg. loss 80.40, avg. ppl 34.97 cum. examples 4958, speed 1486.94 words/sec, time elapsed 4555.57 sec\n",
            "epoch 43, iter 17120, avg. loss 68.97, avg. ppl 30.37 cum. examples 5118, speed 1486.91 words/sec, time elapsed 4557.74 sec\n",
            "epoch 43, iter 17130, avg. loss 70.76, avg. ppl 33.58 cum. examples 5278, speed 1490.48 words/sec, time elapsed 4559.90 sec\n",
            "epoch 43, iter 17140, avg. loss 90.10, avg. ppl 41.23 cum. examples 5438, speed 1538.27 words/sec, time elapsed 4562.42 sec\n",
            "epoch 43, iter 17150, avg. loss 86.30, avg. ppl 38.89 cum. examples 5598, speed 1292.62 words/sec, time elapsed 4565.34 sec\n",
            "epoch 43, iter 17160, avg. loss 85.00, avg. ppl 36.49 cum. examples 5758, speed 1210.59 words/sec, time elapsed 4568.46 sec\n",
            "epoch 43, iter 17170, avg. loss 78.54, avg. ppl 35.01 cum. examples 5918, speed 1418.38 words/sec, time elapsed 4570.95 sec\n",
            "epoch 43, iter 17180, avg. loss 84.75, avg. ppl 37.47 cum. examples 6078, speed 1384.72 words/sec, time elapsed 4573.66 sec\n",
            "epoch 43, iter 17190, avg. loss 72.90, avg. ppl 33.73 cum. examples 6238, speed 1541.87 words/sec, time elapsed 4575.81 sec\n",
            "epoch 43, iter 17200, avg. loss 94.26, avg. ppl 40.28 cum. examples 6398, speed 1272.77 words/sec, time elapsed 4579.01 sec\n",
            "epoch 43, iter 17210, avg. loss 85.13, avg. ppl 35.56 cum. examples 6558, speed 1356.47 words/sec, time elapsed 4581.83 sec\n",
            "epoch 43, iter 17220, avg. loss 74.14, avg. ppl 32.69 cum. examples 6718, speed 1438.04 words/sec, time elapsed 4584.19 sec\n",
            "epoch 43, iter 17230, avg. loss 67.27, avg. ppl 28.23 cum. examples 6878, speed 1535.42 words/sec, time elapsed 4586.29 sec\n",
            "epoch 43, iter 17240, avg. loss 82.43, avg. ppl 37.02 cum. examples 7038, speed 1462.65 words/sec, time elapsed 4588.79 sec\n",
            "epoch 44, iter 17250, avg. loss 78.83, avg. ppl 31.90 cum. examples 7196, speed 1196.52 words/sec, time elapsed 4591.79 sec\n",
            "epoch 44, iter 17260, avg. loss 71.42, avg. ppl 32.41 cum. examples 7356, speed 1512.24 words/sec, time elapsed 4593.97 sec\n",
            "epoch 44, iter 17270, avg. loss 82.92, avg. ppl 33.89 cum. examples 7516, speed 1445.11 words/sec, time elapsed 4596.57 sec\n",
            "epoch 44, iter 17280, avg. loss 80.04, avg. ppl 34.28 cum. examples 7676, speed 1410.98 words/sec, time elapsed 4599.14 sec\n",
            "epoch 44, iter 17290, avg. loss 74.68, avg. ppl 31.81 cum. examples 7836, speed 1463.41 words/sec, time elapsed 4601.50 sec\n",
            "epoch 44, iter 17300, avg. loss 72.26, avg. ppl 30.75 cum. examples 7996, speed 1319.83 words/sec, time elapsed 4604.06 sec\n",
            "epoch 44, iter 17310, avg. loss 83.09, avg. ppl 34.91 cum. examples 8156, speed 1300.79 words/sec, time elapsed 4606.93 sec\n",
            "epoch 44, iter 17320, avg. loss 77.15, avg. ppl 34.09 cum. examples 8316, speed 1273.38 words/sec, time elapsed 4609.68 sec\n",
            "epoch 44, iter 17330, avg. loss 78.38, avg. ppl 31.84 cum. examples 8476, speed 1338.67 words/sec, time elapsed 4612.39 sec\n",
            "epoch 44, iter 17340, avg. loss 72.00, avg. ppl 29.70 cum. examples 8636, speed 1318.88 words/sec, time elapsed 4614.96 sec\n",
            "epoch 44, iter 17350, avg. loss 75.01, avg. ppl 30.19 cum. examples 8796, speed 1620.41 words/sec, time elapsed 4617.14 sec\n",
            "epoch 44, iter 17360, avg. loss 80.78, avg. ppl 33.67 cum. examples 8956, speed 1187.92 words/sec, time elapsed 4620.23 sec\n",
            "epoch 44, iter 17370, avg. loss 69.35, avg. ppl 28.29 cum. examples 9116, speed 1239.33 words/sec, time elapsed 4622.91 sec\n",
            "epoch 44, iter 17380, avg. loss 79.14, avg. ppl 33.15 cum. examples 9276, speed 1309.41 words/sec, time elapsed 4625.67 sec\n",
            "epoch 44, iter 17390, avg. loss 77.49, avg. ppl 33.13 cum. examples 9436, speed 1359.70 words/sec, time elapsed 4628.28 sec\n",
            "epoch 44, iter 17400, avg. loss 74.41, avg. ppl 30.91 cum. examples 9596, speed 1400.96 words/sec, time elapsed 4630.75 sec\n",
            "epoch 44, iter 17410, avg. loss 80.03, avg. ppl 33.58 cum. examples 9756, speed 1238.63 words/sec, time elapsed 4633.70 sec\n",
            "epoch 44, iter 17420, avg. loss 68.70, avg. ppl 26.66 cum. examples 9916, speed 1511.12 words/sec, time elapsed 4635.91 sec\n",
            "epoch 44, iter 17430, avg. loss 78.20, avg. ppl 31.40 cum. examples 10076, speed 1426.34 words/sec, time elapsed 4638.46 sec\n",
            "epoch 44, iter 17440, avg. loss 76.83, avg. ppl 30.00 cum. examples 10236, speed 1444.41 words/sec, time elapsed 4640.96 sec\n",
            "epoch 44, iter 17450, avg. loss 71.82, avg. ppl 29.83 cum. examples 10396, speed 1318.30 words/sec, time elapsed 4643.53 sec\n",
            "epoch 44, iter 17460, avg. loss 72.83, avg. ppl 27.98 cum. examples 10556, speed 1364.41 words/sec, time elapsed 4646.09 sec\n",
            "epoch 44, iter 17470, avg. loss 79.77, avg. ppl 32.76 cum. examples 10716, speed 1383.40 words/sec, time elapsed 4648.73 sec\n",
            "epoch 44, iter 17480, avg. loss 72.85, avg. ppl 30.24 cum. examples 10876, speed 1345.08 words/sec, time elapsed 4651.28 sec\n",
            "epoch 44, iter 17490, avg. loss 89.85, avg. ppl 37.77 cum. examples 11036, speed 1228.20 words/sec, time elapsed 4654.50 sec\n",
            "epoch 44, iter 17500, avg. loss 71.95, avg. ppl 28.88 cum. examples 11196, speed 1632.25 words/sec, time elapsed 4656.60 sec\n",
            "epoch 44, iter 17510, avg. loss 72.82, avg. ppl 28.78 cum. examples 11356, speed 1343.87 words/sec, time elapsed 4659.18 sec\n",
            "epoch 44, iter 17520, avg. loss 73.05, avg. ppl 28.31 cum. examples 11516, speed 1521.32 words/sec, time elapsed 4661.48 sec\n",
            "epoch 44, iter 17530, avg. loss 80.66, avg. ppl 32.78 cum. examples 11676, speed 1455.43 words/sec, time elapsed 4664.02 sec\n",
            "epoch 44, iter 17540, avg. loss 80.25, avg. ppl 30.44 cum. examples 11836, speed 1259.22 words/sec, time elapsed 4667.00 sec\n",
            "epoch 44, iter 17550, avg. loss 74.46, avg. ppl 31.29 cum. examples 11996, speed 1625.25 words/sec, time elapsed 4669.13 sec\n",
            "epoch 44, iter 17560, avg. loss 70.33, avg. ppl 29.90 cum. examples 12156, speed 1299.57 words/sec, time elapsed 4671.68 sec\n",
            "epoch 44, iter 17570, avg. loss 80.52, avg. ppl 31.56 cum. examples 12316, speed 1424.61 words/sec, time elapsed 4674.30 sec\n",
            "epoch 44, iter 17580, avg. loss 78.93, avg. ppl 31.60 cum. examples 12476, speed 1394.30 words/sec, time elapsed 4676.92 sec\n",
            "epoch 44, iter 17590, avg. loss 70.08, avg. ppl 30.65 cum. examples 12636, speed 1406.84 words/sec, time elapsed 4679.25 sec\n",
            "epoch 44, iter 17600, avg. loss 82.35, avg. ppl 33.69 cum. examples 12796, speed 1366.30 words/sec, time elapsed 4681.99 sec\n",
            "epoch 44, iter 17600, cum. loss 78.18, cum. ppl 34.10 cum. examples 12796\n",
            "begin validation ...\n",
            "validation: iter 17600, dev. ppl 55.527201\n",
            "save currently the best model to [results/en-az/model.bin]\n",
            "epoch 44, iter 17610, avg. loss 70.43, avg. ppl 29.01 cum. examples 160, speed 381.97 words/sec, time elapsed 4690.75 sec\n",
            "epoch 44, iter 17620, avg. loss 73.38, avg. ppl 28.99 cum. examples 320, speed 1366.22 words/sec, time elapsed 4693.30 sec\n",
            "epoch 44, iter 17630, avg. loss 67.68, avg. ppl 27.30 cum. examples 480, speed 1306.62 words/sec, time elapsed 4695.81 sec\n",
            "epoch 44, iter 17640, avg. loss 77.52, avg. ppl 29.31 cum. examples 640, speed 1468.28 words/sec, time elapsed 4698.31 sec\n",
            "epoch 45, iter 17650, avg. loss 67.03, avg. ppl 25.97 cum. examples 798, speed 1307.56 words/sec, time elapsed 4700.80 sec\n",
            "epoch 45, iter 17660, avg. loss 67.09, avg. ppl 25.54 cum. examples 958, speed 1467.16 words/sec, time elapsed 4703.06 sec\n",
            "epoch 45, iter 17670, avg. loss 80.71, avg. ppl 31.08 cum. examples 1118, speed 1509.69 words/sec, time elapsed 4705.55 sec\n",
            "epoch 45, iter 17680, avg. loss 72.34, avg. ppl 27.30 cum. examples 1278, speed 1418.80 words/sec, time elapsed 4708.01 sec\n",
            "epoch 45, iter 17690, avg. loss 70.82, avg. ppl 25.54 cum. examples 1438, speed 1356.26 words/sec, time elapsed 4710.59 sec\n",
            "epoch 45, iter 17700, avg. loss 83.31, avg. ppl 30.79 cum. examples 1598, speed 1284.16 words/sec, time elapsed 4713.62 sec\n",
            "epoch 45, iter 17710, avg. loss 71.56, avg. ppl 27.49 cum. examples 1758, speed 1377.97 words/sec, time elapsed 4716.13 sec\n",
            "epoch 45, iter 17720, avg. loss 82.00, avg. ppl 27.68 cum. examples 1918, speed 1476.07 words/sec, time elapsed 4718.80 sec\n",
            "epoch 45, iter 17730, avg. loss 77.55, avg. ppl 29.40 cum. examples 2078, speed 1441.62 words/sec, time elapsed 4721.35 sec\n",
            "epoch 45, iter 17740, avg. loss 73.48, avg. ppl 30.37 cum. examples 2238, speed 1286.91 words/sec, time elapsed 4724.03 sec\n",
            "epoch 45, iter 17750, avg. loss 70.16, avg. ppl 26.41 cum. examples 2398, speed 1327.98 words/sec, time elapsed 4726.61 sec\n",
            "epoch 45, iter 17760, avg. loss 76.77, avg. ppl 29.00 cum. examples 2558, speed 1489.10 words/sec, time elapsed 4729.06 sec\n",
            "epoch 45, iter 17770, avg. loss 69.69, avg. ppl 26.26 cum. examples 2718, speed 1280.59 words/sec, time elapsed 4731.72 sec\n",
            "epoch 45, iter 17780, avg. loss 70.21, avg. ppl 25.00 cum. examples 2878, speed 1458.76 words/sec, time elapsed 4734.12 sec\n",
            "epoch 45, iter 17790, avg. loss 71.42, avg. ppl 27.71 cum. examples 3038, speed 1292.41 words/sec, time elapsed 4736.78 sec\n",
            "epoch 45, iter 17800, avg. loss 71.50, avg. ppl 27.60 cum. examples 3198, speed 1264.59 words/sec, time elapsed 4739.50 sec\n",
            "epoch 45, iter 17810, avg. loss 73.92, avg. ppl 29.24 cum. examples 3358, speed 1399.78 words/sec, time elapsed 4742.01 sec\n",
            "epoch 45, iter 17820, avg. loss 75.27, avg. ppl 29.43 cum. examples 3518, speed 1371.60 words/sec, time elapsed 4744.60 sec\n",
            "epoch 45, iter 17830, avg. loss 74.07, avg. ppl 30.10 cum. examples 3678, speed 1245.35 words/sec, time elapsed 4747.40 sec\n",
            "epoch 45, iter 17840, avg. loss 76.13, avg. ppl 26.70 cum. examples 3838, speed 1350.19 words/sec, time elapsed 4750.14 sec\n",
            "epoch 45, iter 17850, avg. loss 77.15, avg. ppl 28.06 cum. examples 3998, speed 1439.90 words/sec, time elapsed 4752.72 sec\n",
            "epoch 45, iter 17860, avg. loss 75.25, avg. ppl 28.04 cum. examples 4158, speed 1425.96 words/sec, time elapsed 4755.25 sec\n",
            "epoch 45, iter 17870, avg. loss 71.50, avg. ppl 27.10 cum. examples 4318, speed 1420.57 words/sec, time elapsed 4757.69 sec\n",
            "epoch 45, iter 17880, avg. loss 77.65, avg. ppl 29.85 cum. examples 4478, speed 1437.73 words/sec, time elapsed 4760.23 sec\n",
            "epoch 45, iter 17890, avg. loss 74.62, avg. ppl 29.70 cum. examples 4638, speed 1410.89 words/sec, time elapsed 4762.73 sec\n",
            "epoch 45, iter 17900, avg. loss 81.57, avg. ppl 32.26 cum. examples 4798, speed 1200.65 words/sec, time elapsed 4765.86 sec\n",
            "epoch 45, iter 17910, avg. loss 72.45, avg. ppl 27.70 cum. examples 4958, speed 1253.95 words/sec, time elapsed 4768.64 sec\n",
            "epoch 45, iter 17920, avg. loss 67.42, avg. ppl 26.55 cum. examples 5118, speed 1615.83 words/sec, time elapsed 4770.68 sec\n",
            "epoch 45, iter 17930, avg. loss 72.34, avg. ppl 26.01 cum. examples 5278, speed 1371.34 words/sec, time elapsed 4773.27 sec\n",
            "epoch 45, iter 17940, avg. loss 75.67, avg. ppl 29.17 cum. examples 5438, speed 1287.33 words/sec, time elapsed 4776.06 sec\n",
            "epoch 45, iter 17950, avg. loss 70.18, avg. ppl 25.62 cum. examples 5598, speed 1236.36 words/sec, time elapsed 4778.86 sec\n",
            "epoch 45, iter 17960, avg. loss 77.33, avg. ppl 29.25 cum. examples 5758, speed 1523.49 words/sec, time elapsed 4781.26 sec\n",
            "epoch 45, iter 17970, avg. loss 68.78, avg. ppl 25.41 cum. examples 5918, speed 1694.19 words/sec, time elapsed 4783.27 sec\n",
            "epoch 45, iter 17980, avg. loss 72.14, avg. ppl 29.03 cum. examples 6078, speed 1247.17 words/sec, time elapsed 4786.02 sec\n",
            "epoch 45, iter 17990, avg. loss 66.82, avg. ppl 24.27 cum. examples 6238, speed 1475.98 words/sec, time elapsed 4788.29 sec\n",
            "epoch 45, iter 18000, avg. loss 76.54, avg. ppl 29.68 cum. examples 6398, speed 1420.61 words/sec, time elapsed 4790.83 sec\n",
            "epoch 45, iter 18010, avg. loss 65.62, avg. ppl 23.81 cum. examples 6558, speed 1412.26 words/sec, time elapsed 4793.18 sec\n",
            "epoch 45, iter 18020, avg. loss 75.35, avg. ppl 27.00 cum. examples 6718, speed 1247.87 words/sec, time elapsed 4796.11 sec\n",
            "epoch 45, iter 18030, avg. loss 73.54, avg. ppl 26.56 cum. examples 6878, speed 1496.32 words/sec, time elapsed 4798.51 sec\n",
            "epoch 45, iter 18040, avg. loss 66.31, avg. ppl 26.54 cum. examples 7038, speed 1519.41 words/sec, time elapsed 4800.64 sec\n",
            "epoch 46, iter 18050, avg. loss 75.92, avg. ppl 28.20 cum. examples 7196, speed 1209.03 words/sec, time elapsed 4803.61 sec\n",
            "epoch 46, iter 18060, avg. loss 65.17, avg. ppl 23.18 cum. examples 7356, speed 1403.31 words/sec, time elapsed 4805.97 sec\n",
            "epoch 46, iter 18070, avg. loss 66.38, avg. ppl 23.29 cum. examples 7516, speed 1428.80 words/sec, time elapsed 4808.33 sec\n",
            "epoch 46, iter 18080, avg. loss 75.06, avg. ppl 23.98 cum. examples 7676, speed 1312.61 words/sec, time elapsed 4811.21 sec\n",
            "epoch 46, iter 18090, avg. loss 63.62, avg. ppl 22.07 cum. examples 7836, speed 1417.89 words/sec, time elapsed 4813.53 sec\n",
            "epoch 46, iter 18100, avg. loss 77.75, avg. ppl 25.99 cum. examples 7996, speed 1259.87 words/sec, time elapsed 4816.56 sec\n",
            "epoch 46, iter 18110, avg. loss 64.82, avg. ppl 22.31 cum. examples 8156, speed 1478.83 words/sec, time elapsed 4818.82 sec\n",
            "epoch 46, iter 18120, avg. loss 68.10, avg. ppl 25.79 cum. examples 8316, speed 1486.28 words/sec, time elapsed 4821.08 sec\n",
            "epoch 46, iter 18130, avg. loss 68.52, avg. ppl 24.93 cum. examples 8476, speed 1205.39 words/sec, time elapsed 4823.91 sec\n",
            "epoch 46, iter 18140, avg. loss 79.40, avg. ppl 27.60 cum. examples 8636, speed 1491.89 words/sec, time elapsed 4826.47 sec\n",
            "epoch 46, iter 18150, avg. loss 74.88, avg. ppl 25.64 cum. examples 8796, speed 1356.08 words/sec, time elapsed 4829.20 sec\n",
            "epoch 46, iter 18160, avg. loss 75.54, avg. ppl 25.26 cum. examples 8956, speed 1269.15 words/sec, time elapsed 4832.15 sec\n",
            "epoch 46, iter 18170, avg. loss 70.83, avg. ppl 24.07 cum. examples 9116, speed 1249.57 words/sec, time elapsed 4835.00 sec\n",
            "epoch 46, iter 18180, avg. loss 76.51, avg. ppl 28.25 cum. examples 9276, speed 1332.45 words/sec, time elapsed 4837.75 sec\n",
            "epoch 46, iter 18190, avg. loss 75.13, avg. ppl 26.01 cum. examples 9436, speed 1284.60 words/sec, time elapsed 4840.62 sec\n",
            "epoch 46, iter 18200, avg. loss 70.64, avg. ppl 23.56 cum. examples 9596, speed 1347.29 words/sec, time elapsed 4843.27 sec\n",
            "epoch 46, iter 18210, avg. loss 67.78, avg. ppl 24.33 cum. examples 9756, speed 1273.90 words/sec, time elapsed 4845.94 sec\n",
            "epoch 46, iter 18220, avg. loss 64.43, avg. ppl 21.43 cum. examples 9916, speed 1410.96 words/sec, time elapsed 4848.33 sec\n",
            "epoch 46, iter 18230, avg. loss 63.22, avg. ppl 22.77 cum. examples 10076, speed 1222.71 words/sec, time elapsed 4850.97 sec\n",
            "epoch 46, iter 18240, avg. loss 65.71, avg. ppl 22.88 cum. examples 10236, speed 1356.36 words/sec, time elapsed 4853.45 sec\n",
            "epoch 46, iter 18250, avg. loss 65.80, avg. ppl 24.32 cum. examples 10396, speed 1172.25 words/sec, time elapsed 4856.26 sec\n",
            "epoch 46, iter 18260, avg. loss 64.86, avg. ppl 23.02 cum. examples 10556, speed 1506.25 words/sec, time elapsed 4858.46 sec\n",
            "epoch 46, iter 18270, avg. loss 62.94, avg. ppl 23.20 cum. examples 10716, speed 1457.50 words/sec, time elapsed 4860.66 sec\n",
            "epoch 46, iter 18280, avg. loss 76.59, avg. ppl 26.63 cum. examples 10876, speed 1282.31 words/sec, time elapsed 4863.57 sec\n",
            "epoch 46, iter 18290, avg. loss 78.79, avg. ppl 28.13 cum. examples 11036, speed 1169.66 words/sec, time elapsed 4866.80 sec\n",
            "epoch 46, iter 18300, avg. loss 69.88, avg. ppl 25.74 cum. examples 11196, speed 1630.29 words/sec, time elapsed 4868.91 sec\n",
            "epoch 46, iter 18310, avg. loss 69.49, avg. ppl 24.31 cum. examples 11356, speed 1307.41 words/sec, time elapsed 4871.58 sec\n",
            "epoch 46, iter 18320, avg. loss 74.87, avg. ppl 26.99 cum. examples 11516, speed 1341.91 words/sec, time elapsed 4874.28 sec\n",
            "epoch 46, iter 18330, avg. loss 75.21, avg. ppl 25.94 cum. examples 11676, speed 1557.16 words/sec, time elapsed 4876.66 sec\n",
            "epoch 46, iter 18340, avg. loss 74.76, avg. ppl 27.59 cum. examples 11836, speed 1303.73 words/sec, time elapsed 4879.42 sec\n",
            "epoch 46, iter 18350, avg. loss 68.25, avg. ppl 25.42 cum. examples 11996, speed 1225.58 words/sec, time elapsed 4882.18 sec\n",
            "epoch 46, iter 18360, avg. loss 70.16, avg. ppl 24.31 cum. examples 12156, speed 1399.61 words/sec, time elapsed 4884.69 sec\n",
            "epoch 46, iter 18370, avg. loss 81.36, avg. ppl 28.27 cum. examples 12316, speed 1350.74 words/sec, time elapsed 4887.58 sec\n",
            "epoch 46, iter 18380, avg. loss 81.99, avg. ppl 29.33 cum. examples 12476, speed 1417.48 words/sec, time elapsed 4890.31 sec\n",
            "epoch 46, iter 18390, avg. loss 76.73, avg. ppl 28.67 cum. examples 12636, speed 1393.56 words/sec, time elapsed 4892.94 sec\n",
            "epoch 46, iter 18400, avg. loss 67.11, avg. ppl 22.17 cum. examples 12796, speed 1482.76 words/sec, time elapsed 4895.28 sec\n",
            "epoch 46, iter 18400, cum. loss 72.38, cum. ppl 26.58 cum. examples 12796\n",
            "begin validation ...\n",
            "validation: iter 18400, dev. ppl 58.569115\n",
            "hit patience 1\n",
            "epoch 46, iter 18410, avg. loss 68.84, avg. ppl 22.06 cum. examples 160, speed 458.67 words/sec, time elapsed 4903.04 sec\n",
            "epoch 46, iter 18420, avg. loss 77.43, avg. ppl 25.23 cum. examples 320, speed 1241.03 words/sec, time elapsed 4906.13 sec\n",
            "epoch 46, iter 18430, avg. loss 59.79, avg. ppl 21.15 cum. examples 480, speed 1702.03 words/sec, time elapsed 4907.97 sec\n",
            "epoch 46, iter 18440, avg. loss 72.10, avg. ppl 25.78 cum. examples 640, speed 1353.65 words/sec, time elapsed 4910.60 sec\n",
            "epoch 47, iter 18450, avg. loss 71.10, avg. ppl 24.01 cum. examples 798, speed 1328.74 words/sec, time elapsed 4913.26 sec\n",
            "epoch 47, iter 18460, avg. loss 65.74, avg. ppl 20.42 cum. examples 958, speed 986.02 words/sec, time elapsed 4916.79 sec\n",
            "epoch 47, iter 18470, avg. loss 63.10, avg. ppl 20.44 cum. examples 1118, speed 1330.59 words/sec, time elapsed 4919.31 sec\n",
            "epoch 47, iter 18480, avg. loss 57.22, avg. ppl 18.33 cum. examples 1278, speed 1538.27 words/sec, time elapsed 4921.35 sec\n",
            "epoch 47, iter 18490, avg. loss 66.08, avg. ppl 22.43 cum. examples 1438, speed 1364.17 words/sec, time elapsed 4923.84 sec\n",
            "epoch 47, iter 18500, avg. loss 69.91, avg. ppl 23.80 cum. examples 1598, speed 1340.72 words/sec, time elapsed 4926.48 sec\n",
            "epoch 47, iter 18510, avg. loss 68.78, avg. ppl 23.08 cum. examples 1758, speed 1438.31 words/sec, time elapsed 4928.91 sec\n",
            "epoch 47, iter 18520, avg. loss 72.37, avg. ppl 24.27 cum. examples 1918, speed 1245.87 words/sec, time elapsed 4931.83 sec\n",
            "epoch 47, iter 18530, avg. loss 65.06, avg. ppl 21.46 cum. examples 2078, speed 1563.30 words/sec, time elapsed 4934.00 sec\n",
            "epoch 47, iter 18540, avg. loss 77.54, avg. ppl 24.91 cum. examples 2238, speed 1457.79 words/sec, time elapsed 4936.65 sec\n",
            "epoch 47, iter 18550, avg. loss 66.38, avg. ppl 21.76 cum. examples 2398, speed 1423.93 words/sec, time elapsed 4939.07 sec\n",
            "epoch 47, iter 18560, avg. loss 69.55, avg. ppl 21.16 cum. examples 2558, speed 1457.08 words/sec, time elapsed 4941.57 sec\n",
            "epoch 47, iter 18570, avg. loss 65.23, avg. ppl 20.45 cum. examples 2718, speed 1589.66 words/sec, time elapsed 4943.75 sec\n",
            "epoch 47, iter 18580, avg. loss 68.02, avg. ppl 21.98 cum. examples 2878, speed 1359.09 words/sec, time elapsed 4946.34 sec\n",
            "epoch 47, iter 18590, avg. loss 67.62, avg. ppl 22.48 cum. examples 3038, speed 1521.01 words/sec, time elapsed 4948.62 sec\n",
            "epoch 47, iter 18600, avg. loss 64.80, avg. ppl 21.05 cum. examples 3198, speed 1276.02 words/sec, time elapsed 4951.29 sec\n",
            "epoch 47, iter 18610, avg. loss 65.78, avg. ppl 21.36 cum. examples 3358, speed 1398.91 words/sec, time elapsed 4953.75 sec\n",
            "epoch 47, iter 18620, avg. loss 59.48, avg. ppl 18.80 cum. examples 3518, speed 1579.39 words/sec, time elapsed 4955.80 sec\n",
            "epoch 47, iter 18630, avg. loss 72.67, avg. ppl 22.49 cum. examples 3678, speed 1224.39 words/sec, time elapsed 4958.85 sec\n",
            "epoch 47, iter 18640, avg. loss 66.18, avg. ppl 21.58 cum. examples 3838, speed 1256.00 words/sec, time elapsed 4961.60 sec\n",
            "epoch 47, iter 18650, avg. loss 77.44, avg. ppl 25.24 cum. examples 3998, speed 1468.20 words/sec, time elapsed 4964.21 sec\n",
            "epoch 47, iter 18660, avg. loss 69.74, avg. ppl 22.57 cum. examples 4158, speed 1345.79 words/sec, time elapsed 4966.87 sec\n",
            "epoch 47, iter 18670, avg. loss 66.80, avg. ppl 22.50 cum. examples 4318, speed 1704.34 words/sec, time elapsed 4968.89 sec\n",
            "epoch 47, iter 18680, avg. loss 64.92, avg. ppl 21.46 cum. examples 4478, speed 1572.60 words/sec, time elapsed 4971.04 sec\n",
            "epoch 47, iter 18690, avg. loss 65.71, avg. ppl 20.77 cum. examples 4638, speed 1522.91 words/sec, time elapsed 4973.32 sec\n",
            "epoch 47, iter 18700, avg. loss 66.06, avg. ppl 21.41 cum. examples 4798, speed 1236.35 words/sec, time elapsed 4976.11 sec\n",
            "epoch 47, iter 18710, avg. loss 71.11, avg. ppl 24.04 cum. examples 4958, speed 1358.49 words/sec, time elapsed 4978.74 sec\n",
            "epoch 47, iter 18720, avg. loss 69.73, avg. ppl 22.84 cum. examples 5118, speed 1283.90 words/sec, time elapsed 4981.52 sec\n",
            "epoch 47, iter 18730, avg. loss 72.04, avg. ppl 23.95 cum. examples 5278, speed 1444.53 words/sec, time elapsed 4984.03 sec\n",
            "epoch 47, iter 18740, avg. loss 70.88, avg. ppl 24.01 cum. examples 5438, speed 1361.30 words/sec, time elapsed 4986.65 sec\n",
            "epoch 47, iter 18750, avg. loss 66.84, avg. ppl 23.33 cum. examples 5598, speed 1375.11 words/sec, time elapsed 4989.12 sec\n",
            "epoch 47, iter 18760, avg. loss 68.85, avg. ppl 22.66 cum. examples 5758, speed 1409.38 words/sec, time elapsed 4991.63 sec\n",
            "epoch 47, iter 18770, avg. loss 67.99, avg. ppl 21.89 cum. examples 5918, speed 1337.83 words/sec, time elapsed 4994.26 sec\n",
            "epoch 47, iter 18780, avg. loss 75.95, avg. ppl 22.68 cum. examples 6078, speed 1407.86 words/sec, time elapsed 4997.03 sec\n",
            "epoch 47, iter 18790, avg. loss 73.54, avg. ppl 24.75 cum. examples 6238, speed 1200.63 words/sec, time elapsed 5000.08 sec\n",
            "epoch 47, iter 18800, avg. loss 55.69, avg. ppl 17.86 cum. examples 6398, speed 1316.09 words/sec, time elapsed 5002.43 sec\n",
            "epoch 47, iter 18810, avg. loss 82.48, avg. ppl 28.18 cum. examples 6558, speed 1247.71 words/sec, time elapsed 5005.60 sec\n",
            "epoch 47, iter 18820, avg. loss 75.21, avg. ppl 24.46 cum. examples 6718, speed 1310.27 words/sec, time elapsed 5008.47 sec\n",
            "epoch 47, iter 18830, avg. loss 63.36, avg. ppl 21.63 cum. examples 6878, speed 1258.49 words/sec, time elapsed 5011.09 sec\n",
            "epoch 47, iter 18840, avg. loss 77.27, avg. ppl 25.13 cum. examples 7038, speed 1256.22 words/sec, time elapsed 5014.14 sec\n",
            "epoch 48, iter 18850, avg. loss 80.59, avg. ppl 26.55 cum. examples 7196, speed 1417.87 words/sec, time elapsed 5016.88 sec\n",
            "epoch 48, iter 18860, avg. loss 68.92, avg. ppl 20.21 cum. examples 7356, speed 1444.71 words/sec, time elapsed 5019.42 sec\n",
            "epoch 48, iter 18870, avg. loss 55.12, avg. ppl 16.78 cum. examples 7516, speed 1378.17 words/sec, time elapsed 5021.69 sec\n",
            "epoch 48, iter 18880, avg. loss 68.47, avg. ppl 21.72 cum. examples 7676, speed 1442.34 words/sec, time elapsed 5024.16 sec\n",
            "epoch 48, iter 18890, avg. loss 70.37, avg. ppl 21.46 cum. examples 7836, speed 1294.13 words/sec, time elapsed 5026.99 sec\n",
            "epoch 48, iter 18900, avg. loss 66.43, avg. ppl 20.52 cum. examples 7996, speed 1453.74 words/sec, time elapsed 5029.41 sec\n",
            "epoch 48, iter 18910, avg. loss 60.22, avg. ppl 17.42 cum. examples 8156, speed 1401.53 words/sec, time elapsed 5031.82 sec\n",
            "epoch 48, iter 18920, avg. loss 61.76, avg. ppl 21.11 cum. examples 8316, speed 1377.29 words/sec, time elapsed 5034.17 sec\n",
            "epoch 48, iter 18930, avg. loss 71.45, avg. ppl 20.86 cum. examples 8476, speed 1316.91 words/sec, time elapsed 5037.03 sec\n",
            "epoch 48, iter 18940, avg. loss 73.41, avg. ppl 23.15 cum. examples 8636, speed 1115.54 words/sec, time elapsed 5040.38 sec\n",
            "epoch 48, iter 18950, avg. loss 65.35, avg. ppl 20.23 cum. examples 8796, speed 1478.54 words/sec, time elapsed 5042.73 sec\n",
            "epoch 48, iter 18960, avg. loss 68.19, avg. ppl 19.32 cum. examples 8956, speed 1385.31 words/sec, time elapsed 5045.39 sec\n",
            "epoch 48, iter 18970, avg. loss 67.31, avg. ppl 20.22 cum. examples 9116, speed 1431.40 words/sec, time elapsed 5047.90 sec\n",
            "epoch 48, iter 18980, avg. loss 74.79, avg. ppl 22.42 cum. examples 9276, speed 1257.58 words/sec, time elapsed 5050.96 sec\n",
            "epoch 48, iter 18990, avg. loss 69.28, avg. ppl 20.31 cum. examples 9436, speed 1286.26 words/sec, time elapsed 5053.82 sec\n",
            "epoch 48, iter 19000, avg. loss 65.25, avg. ppl 19.71 cum. examples 9596, speed 1301.70 words/sec, time elapsed 5056.51 sec\n",
            "epoch 48, iter 19010, avg. loss 58.75, avg. ppl 18.05 cum. examples 9756, speed 1461.51 words/sec, time elapsed 5058.73 sec\n",
            "epoch 48, iter 19020, avg. loss 66.46, avg. ppl 19.55 cum. examples 9916, speed 1318.27 words/sec, time elapsed 5061.44 sec\n",
            "epoch 48, iter 19030, avg. loss 58.63, avg. ppl 17.09 cum. examples 10076, speed 1460.77 words/sec, time elapsed 5063.71 sec\n",
            "epoch 48, iter 19040, avg. loss 76.24, avg. ppl 24.28 cum. examples 10236, speed 1271.02 words/sec, time elapsed 5066.72 sec\n",
            "epoch 48, iter 19050, avg. loss 67.00, avg. ppl 19.77 cum. examples 10396, speed 1229.06 words/sec, time elapsed 5069.64 sec\n",
            "epoch 48, iter 19060, avg. loss 69.24, avg. ppl 21.94 cum. examples 10556, speed 1256.96 words/sec, time elapsed 5072.49 sec\n",
            "epoch 48, iter 19070, avg. loss 66.78, avg. ppl 21.13 cum. examples 10716, speed 1536.66 words/sec, time elapsed 5074.77 sec\n",
            "epoch 48, iter 19080, avg. loss 67.14, avg. ppl 19.95 cum. examples 10876, speed 1436.29 words/sec, time elapsed 5077.27 sec\n",
            "epoch 48, iter 19090, avg. loss 77.74, avg. ppl 24.31 cum. examples 11036, speed 1161.80 words/sec, time elapsed 5080.62 sec\n",
            "epoch 48, iter 19100, avg. loss 57.87, avg. ppl 18.04 cum. examples 11196, speed 1612.84 words/sec, time elapsed 5082.61 sec\n",
            "epoch 48, iter 19110, avg. loss 72.14, avg. ppl 21.26 cum. examples 11356, speed 1283.02 words/sec, time elapsed 5085.55 sec\n",
            "epoch 48, iter 19120, avg. loss 67.19, avg. ppl 21.43 cum. examples 11516, speed 1330.39 words/sec, time elapsed 5088.19 sec\n",
            "epoch 48, iter 19130, avg. loss 69.06, avg. ppl 21.24 cum. examples 11676, speed 1318.35 words/sec, time elapsed 5090.93 sec\n",
            "epoch 48, iter 19140, avg. loss 62.34, avg. ppl 19.24 cum. examples 11836, speed 1610.23 words/sec, time elapsed 5093.03 sec\n",
            "epoch 48, iter 19150, avg. loss 65.99, avg. ppl 19.98 cum. examples 11996, speed 1362.25 words/sec, time elapsed 5095.62 sec\n",
            "epoch 48, iter 19160, avg. loss 70.98, avg. ppl 21.51 cum. examples 12156, speed 1475.84 words/sec, time elapsed 5098.12 sec\n",
            "epoch 48, iter 19170, avg. loss 58.67, avg. ppl 18.64 cum. examples 12316, speed 1516.25 words/sec, time elapsed 5100.24 sec\n",
            "epoch 48, iter 19180, avg. loss 61.60, avg. ppl 19.50 cum. examples 12476, speed 1484.58 words/sec, time elapsed 5102.47 sec\n",
            "epoch 48, iter 19190, avg. loss 71.35, avg. ppl 19.90 cum. examples 12636, speed 1377.18 words/sec, time elapsed 5105.25 sec\n",
            "epoch 48, iter 19200, avg. loss 62.69, avg. ppl 18.82 cum. examples 12796, speed 1274.35 words/sec, time elapsed 5107.93 sec\n",
            "epoch 48, iter 19200, cum. loss 67.96, cum. ppl 21.59 cum. examples 12796\n",
            "begin validation ...\n",
            "validation: iter 19200, dev. ppl 63.537649\n",
            "hit patience 2\n",
            "epoch 48, iter 19210, avg. loss 69.34, avg. ppl 20.54 cum. examples 160, speed 440.45 words/sec, time elapsed 5116.26 sec\n",
            "epoch 48, iter 19220, avg. loss 63.53, avg. ppl 19.36 cum. examples 320, speed 1218.49 words/sec, time elapsed 5119.08 sec\n",
            "epoch 48, iter 19230, avg. loss 63.63, avg. ppl 20.44 cum. examples 480, speed 1366.27 words/sec, time elapsed 5121.55 sec\n",
            "epoch 48, iter 19240, avg. loss 62.89, avg. ppl 19.27 cum. examples 640, speed 1415.97 words/sec, time elapsed 5123.95 sec\n",
            "epoch 49, iter 19250, avg. loss 66.04, avg. ppl 19.63 cum. examples 798, speed 1401.33 words/sec, time elapsed 5126.45 sec\n",
            "epoch 49, iter 19260, avg. loss 59.69, avg. ppl 16.83 cum. examples 958, speed 1488.07 words/sec, time elapsed 5128.72 sec\n",
            "epoch 49, iter 19270, avg. loss 58.58, avg. ppl 16.96 cum. examples 1118, speed 1560.78 words/sec, time elapsed 5130.85 sec\n",
            "epoch 49, iter 19280, avg. loss 67.59, avg. ppl 17.56 cum. examples 1278, speed 1340.37 words/sec, time elapsed 5133.66 sec\n",
            "epoch 49, iter 19290, avg. loss 63.88, avg. ppl 18.39 cum. examples 1438, speed 1466.63 words/sec, time elapsed 5136.06 sec\n",
            "epoch 49, iter 19300, avg. loss 66.74, avg. ppl 17.99 cum. examples 1598, speed 1193.56 words/sec, time elapsed 5139.15 sec\n",
            "epoch 49, iter 19310, avg. loss 54.78, avg. ppl 15.50 cum. examples 1758, speed 1424.50 words/sec, time elapsed 5141.40 sec\n",
            "epoch 49, iter 19320, avg. loss 62.56, avg. ppl 18.96 cum. examples 1918, speed 1247.72 words/sec, time elapsed 5144.12 sec\n",
            "epoch 49, iter 19330, avg. loss 65.69, avg. ppl 18.28 cum. examples 2078, speed 1469.05 words/sec, time elapsed 5146.59 sec\n",
            "epoch 49, iter 19340, avg. loss 67.09, avg. ppl 17.96 cum. examples 2238, speed 1234.24 words/sec, time elapsed 5149.60 sec\n",
            "epoch 49, iter 19350, avg. loss 66.57, avg. ppl 21.12 cum. examples 2398, speed 1607.04 words/sec, time elapsed 5151.77 sec\n",
            "epoch 49, iter 19360, avg. loss 66.39, avg. ppl 18.30 cum. examples 2558, speed 1245.92 words/sec, time elapsed 5154.70 sec\n",
            "epoch 49, iter 19370, avg. loss 69.80, avg. ppl 20.02 cum. examples 2718, speed 1318.68 words/sec, time elapsed 5157.53 sec\n",
            "epoch 49, iter 19380, avg. loss 62.46, avg. ppl 18.33 cum. examples 2878, speed 1304.91 words/sec, time elapsed 5160.16 sec\n",
            "epoch 49, iter 19390, avg. loss 60.17, avg. ppl 17.89 cum. examples 3038, speed 1517.28 words/sec, time elapsed 5162.36 sec\n",
            "epoch 49, iter 19400, avg. loss 60.44, avg. ppl 16.85 cum. examples 3198, speed 1424.23 words/sec, time elapsed 5164.77 sec\n",
            "epoch 49, iter 19410, avg. loss 61.43, avg. ppl 17.29 cum. examples 3358, speed 1321.95 words/sec, time elapsed 5167.38 sec\n",
            "epoch 49, iter 19420, avg. loss 69.26, avg. ppl 20.19 cum. examples 3518, speed 1300.33 words/sec, time elapsed 5170.21 sec\n",
            "epoch 49, iter 19430, avg. loss 65.90, avg. ppl 19.07 cum. examples 3678, speed 1349.52 words/sec, time elapsed 5172.86 sec\n",
            "epoch 49, iter 19440, avg. loss 71.03, avg. ppl 21.09 cum. examples 3838, speed 1320.59 words/sec, time elapsed 5175.68 sec\n",
            "epoch 49, iter 19450, avg. loss 61.48, avg. ppl 16.93 cum. examples 3998, speed 1467.90 words/sec, time elapsed 5178.05 sec\n",
            "epoch 49, iter 19460, avg. loss 71.45, avg. ppl 19.35 cum. examples 4158, speed 1286.79 words/sec, time elapsed 5181.05 sec\n",
            "epoch 49, iter 19470, avg. loss 57.17, avg. ppl 15.81 cum. examples 4318, speed 1391.24 words/sec, time elapsed 5183.43 sec\n",
            "epoch 49, iter 19480, avg. loss 61.73, avg. ppl 18.74 cum. examples 4478, speed 1337.04 words/sec, time elapsed 5185.95 sec\n",
            "epoch 49, iter 19490, avg. loss 62.58, avg. ppl 17.56 cum. examples 4638, speed 1465.81 words/sec, time elapsed 5188.34 sec\n",
            "epoch 49, iter 19500, avg. loss 71.47, avg. ppl 20.21 cum. examples 4798, speed 1363.99 words/sec, time elapsed 5191.13 sec\n",
            "epoch 49, iter 19510, avg. loss 61.14, avg. ppl 17.24 cum. examples 4958, speed 1155.52 words/sec, time elapsed 5194.10 sec\n",
            "epoch 49, iter 19520, avg. loss 61.51, avg. ppl 17.60 cum. examples 5118, speed 1515.24 words/sec, time elapsed 5196.36 sec\n",
            "epoch 49, iter 19530, avg. loss 67.93, avg. ppl 19.54 cum. examples 5278, speed 1515.21 words/sec, time elapsed 5198.78 sec\n",
            "epoch 49, iter 19540, avg. loss 60.27, avg. ppl 16.72 cum. examples 5438, speed 1411.01 words/sec, time elapsed 5201.20 sec\n",
            "epoch 49, iter 19550, avg. loss 59.20, avg. ppl 16.72 cum. examples 5598, speed 1355.23 words/sec, time elapsed 5203.69 sec\n",
            "epoch 49, iter 19560, avg. loss 57.88, avg. ppl 16.58 cum. examples 5758, speed 1438.85 words/sec, time elapsed 5205.98 sec\n",
            "epoch 49, iter 19570, avg. loss 76.90, avg. ppl 22.84 cum. examples 5918, speed 1305.85 words/sec, time elapsed 5208.99 sec\n",
            "epoch 49, iter 19580, avg. loss 67.93, avg. ppl 18.56 cum. examples 6078, speed 1578.97 words/sec, time elapsed 5211.35 sec\n",
            "epoch 49, iter 19590, avg. loss 61.57, avg. ppl 17.79 cum. examples 6238, speed 1528.67 words/sec, time elapsed 5213.59 sec\n",
            "epoch 49, iter 19600, avg. loss 65.25, avg. ppl 19.29 cum. examples 6398, speed 1654.38 words/sec, time elapsed 5215.72 sec\n",
            "epoch 49, iter 19610, avg. loss 63.91, avg. ppl 17.33 cum. examples 6558, speed 1116.28 words/sec, time elapsed 5218.93 sec\n",
            "epoch 49, iter 19620, avg. loss 61.44, avg. ppl 17.13 cum. examples 6718, speed 1307.88 words/sec, time elapsed 5221.57 sec\n",
            "epoch 49, iter 19630, avg. loss 70.12, avg. ppl 21.33 cum. examples 6878, speed 1325.93 words/sec, time elapsed 5224.34 sec\n",
            "epoch 49, iter 19640, avg. loss 62.82, avg. ppl 18.51 cum. examples 7038, speed 1453.23 words/sec, time elapsed 5226.71 sec\n",
            "epoch 50, iter 19650, avg. loss 65.99, avg. ppl 18.79 cum. examples 7196, speed 1128.69 words/sec, time elapsed 5229.86 sec\n",
            "epoch 50, iter 19660, avg. loss 62.58, avg. ppl 15.80 cum. examples 7356, speed 1309.55 words/sec, time elapsed 5232.63 sec\n",
            "epoch 50, iter 19670, avg. loss 74.58, avg. ppl 20.06 cum. examples 7516, speed 1436.26 words/sec, time elapsed 5235.40 sec\n",
            "epoch 50, iter 19680, avg. loss 62.73, avg. ppl 16.52 cum. examples 7676, speed 1469.11 words/sec, time elapsed 5237.84 sec\n",
            "epoch 50, iter 19690, avg. loss 58.73, avg. ppl 15.34 cum. examples 7836, speed 1462.28 words/sec, time elapsed 5240.19 sec\n",
            "epoch 50, iter 19700, avg. loss 56.71, avg. ppl 15.06 cum. examples 7996, speed 1605.33 words/sec, time elapsed 5242.27 sec\n",
            "epoch 50, iter 19710, avg. loss 64.34, avg. ppl 17.35 cum. examples 8156, speed 1408.31 words/sec, time elapsed 5244.84 sec\n",
            "epoch 50, iter 19720, avg. loss 62.26, avg. ppl 16.24 cum. examples 8316, speed 1280.28 words/sec, time elapsed 5247.63 sec\n",
            "epoch 50, iter 19730, avg. loss 62.03, avg. ppl 16.12 cum. examples 8476, speed 1486.25 words/sec, time elapsed 5250.03 sec\n",
            "epoch 50, iter 19740, avg. loss 63.36, avg. ppl 17.56 cum. examples 8636, speed 1219.73 words/sec, time elapsed 5252.93 sec\n",
            "epoch 50, iter 19750, avg. loss 59.35, avg. ppl 15.34 cum. examples 8796, speed 1221.73 words/sec, time elapsed 5255.78 sec\n",
            "epoch 50, iter 19760, avg. loss 68.72, avg. ppl 19.17 cum. examples 8956, speed 1319.98 words/sec, time elapsed 5258.60 sec\n",
            "epoch 50, iter 19770, avg. loss 46.55, avg. ppl 13.16 cum. examples 9116, speed 1477.45 words/sec, time elapsed 5260.55 sec\n",
            "epoch 50, iter 19780, avg. loss 64.99, avg. ppl 18.15 cum. examples 9276, speed 1286.17 words/sec, time elapsed 5263.34 sec\n",
            "epoch 50, iter 19790, avg. loss 62.39, avg. ppl 17.20 cum. examples 9436, speed 1665.11 words/sec, time elapsed 5265.45 sec\n",
            "epoch 50, iter 19800, avg. loss 60.83, avg. ppl 15.81 cum. examples 9596, speed 1484.31 words/sec, time elapsed 5267.82 sec\n",
            "epoch 50, iter 19810, avg. loss 59.45, avg. ppl 15.68 cum. examples 9756, speed 1371.93 words/sec, time elapsed 5270.34 sec\n",
            "epoch 50, iter 19820, avg. loss 62.49, avg. ppl 16.11 cum. examples 9916, speed 1394.49 words/sec, time elapsed 5272.92 sec\n",
            "epoch 50, iter 19830, avg. loss 56.66, avg. ppl 14.56 cum. examples 10076, speed 1431.76 words/sec, time elapsed 5275.29 sec\n",
            "epoch 50, iter 19840, avg. loss 63.21, avg. ppl 17.53 cum. examples 10236, speed 1296.57 words/sec, time elapsed 5278.01 sec\n",
            "epoch 50, iter 19850, avg. loss 65.32, avg. ppl 16.87 cum. examples 10396, speed 1280.63 words/sec, time elapsed 5280.90 sec\n",
            "epoch 50, iter 19860, avg. loss 57.71, avg. ppl 15.46 cum. examples 10556, speed 1497.44 words/sec, time elapsed 5283.15 sec\n",
            "epoch 50, iter 19870, avg. loss 65.41, avg. ppl 16.60 cum. examples 10716, speed 1445.85 words/sec, time elapsed 5285.73 sec\n",
            "epoch 50, iter 19880, avg. loss 66.64, avg. ppl 17.36 cum. examples 10876, speed 1224.40 words/sec, time elapsed 5288.78 sec\n",
            "epoch 50, iter 19890, avg. loss 61.84, avg. ppl 16.88 cum. examples 11036, speed 1352.89 words/sec, time elapsed 5291.37 sec\n",
            "epoch 50, iter 19900, avg. loss 68.88, avg. ppl 19.05 cum. examples 11196, speed 1527.99 words/sec, time elapsed 5293.81 sec\n",
            "epoch 50, iter 19910, avg. loss 75.37, avg. ppl 19.41 cum. examples 11356, speed 1426.75 words/sec, time elapsed 5296.66 sec\n",
            "epoch 50, iter 19920, avg. loss 71.35, avg. ppl 17.48 cum. examples 11516, speed 1083.43 words/sec, time elapsed 5300.35 sec\n",
            "epoch 50, iter 19930, avg. loss 60.87, avg. ppl 16.19 cum. examples 11676, speed 1300.26 words/sec, time elapsed 5303.04 sec\n",
            "epoch 50, iter 19940, avg. loss 63.32, avg. ppl 15.96 cum. examples 11836, speed 1522.51 words/sec, time elapsed 5305.44 sec\n",
            "epoch 50, iter 19950, avg. loss 71.83, avg. ppl 19.92 cum. examples 11996, speed 1256.93 words/sec, time elapsed 5308.49 sec\n",
            "epoch 50, iter 19960, avg. loss 59.96, avg. ppl 16.93 cum. examples 12156, speed 1423.86 words/sec, time elapsed 5310.88 sec\n",
            "epoch 50, iter 19970, avg. loss 62.21, avg. ppl 17.18 cum. examples 12316, speed 1413.21 words/sec, time elapsed 5313.35 sec\n",
            "epoch 50, iter 19980, avg. loss 67.56, avg. ppl 19.03 cum. examples 12476, speed 1266.08 words/sec, time elapsed 5316.25 sec\n",
            "epoch 50, iter 19990, avg. loss 55.01, avg. ppl 14.65 cum. examples 12636, speed 1411.76 words/sec, time elapsed 5318.57 sec\n",
            "epoch 50, iter 20000, avg. loss 62.16, avg. ppl 16.50 cum. examples 12796, speed 1683.63 words/sec, time elapsed 5320.68 sec\n",
            "epoch 50, iter 20000, cum. loss 63.78, cum. ppl 17.72 cum. examples 12796\n",
            "begin validation ...\n",
            "validation: iter 20000, dev. ppl 70.111577\n",
            "hit patience 3\n",
            "epoch 50, iter 20010, avg. loss 61.75, avg. ppl 17.02 cum. examples 160, speed 414.53 words/sec, time elapsed 5329.09 sec\n",
            "epoch 50, iter 20020, avg. loss 56.17, avg. ppl 15.80 cum. examples 320, speed 1176.80 words/sec, time elapsed 5331.86 sec\n",
            "epoch 50, iter 20030, avg. loss 54.32, avg. ppl 14.05 cum. examples 480, speed 1411.93 words/sec, time elapsed 5334.19 sec\n",
            "epoch 50, iter 20040, avg. loss 51.69, avg. ppl 13.52 cum. examples 640, speed 1449.61 words/sec, time elapsed 5336.38 sec\n",
            "epoch 50, iter 20050, avg. loss 50.69, avg. ppl 13.25 cum. examples 798, speed 1686.92 words/sec, time elapsed 5338.21 sec\n",
            "epoch 51, iter 20060, avg. loss 55.84, avg. ppl 13.81 cum. examples 958, speed 1396.47 words/sec, time elapsed 5340.65 sec\n",
            "epoch 51, iter 20070, avg. loss 65.24, avg. ppl 16.76 cum. examples 1118, speed 1277.93 words/sec, time elapsed 5343.55 sec\n",
            "epoch 51, iter 20080, avg. loss 57.57, avg. ppl 14.61 cum. examples 1278, speed 1482.00 words/sec, time elapsed 5345.87 sec\n",
            "epoch 51, iter 20090, avg. loss 61.99, avg. ppl 15.67 cum. examples 1438, speed 1264.38 words/sec, time elapsed 5348.72 sec\n",
            "epoch 51, iter 20100, avg. loss 55.31, avg. ppl 13.80 cum. examples 1598, speed 1438.79 words/sec, time elapsed 5351.06 sec\n",
            "epoch 51, iter 20110, avg. loss 53.21, avg. ppl 12.61 cum. examples 1758, speed 1492.01 words/sec, time elapsed 5353.31 sec\n",
            "epoch 51, iter 20120, avg. loss 57.50, avg. ppl 13.72 cum. examples 1918, speed 1528.20 words/sec, time elapsed 5355.61 sec\n",
            "epoch 51, iter 20130, avg. loss 50.44, avg. ppl 12.58 cum. examples 2078, speed 1392.29 words/sec, time elapsed 5357.90 sec\n",
            "epoch 51, iter 20140, avg. loss 55.52, avg. ppl 14.00 cum. examples 2238, speed 1525.56 words/sec, time elapsed 5360.11 sec\n",
            "epoch 51, iter 20150, avg. loss 63.48, avg. ppl 16.09 cum. examples 2398, speed 1355.09 words/sec, time elapsed 5362.81 sec\n",
            "epoch 51, iter 20160, avg. loss 66.08, avg. ppl 16.17 cum. examples 2558, speed 1530.35 words/sec, time elapsed 5365.29 sec\n",
            "epoch 51, iter 20170, avg. loss 59.85, avg. ppl 14.84 cum. examples 2718, speed 1275.38 words/sec, time elapsed 5368.07 sec\n",
            "epoch 51, iter 20180, avg. loss 71.51, avg. ppl 18.53 cum. examples 2878, speed 1309.27 words/sec, time elapsed 5371.06 sec\n",
            "epoch 51, iter 20190, avg. loss 60.92, avg. ppl 14.83 cum. examples 3038, speed 1413.93 words/sec, time elapsed 5373.62 sec\n",
            "epoch 51, iter 20200, avg. loss 66.99, avg. ppl 15.76 cum. examples 3198, speed 1515.32 words/sec, time elapsed 5376.19 sec\n",
            "epoch 51, iter 20210, avg. loss 63.14, avg. ppl 15.66 cum. examples 3358, speed 1352.39 words/sec, time elapsed 5378.90 sec\n",
            "epoch 51, iter 20220, avg. loss 62.79, avg. ppl 16.33 cum. examples 3518, speed 1481.34 words/sec, time elapsed 5381.33 sec\n",
            "epoch 51, iter 20230, avg. loss 57.26, avg. ppl 14.64 cum. examples 3678, speed 1298.69 words/sec, time elapsed 5383.96 sec\n",
            "epoch 51, iter 20240, avg. loss 62.06, avg. ppl 15.04 cum. examples 3838, speed 1314.70 words/sec, time elapsed 5386.75 sec\n",
            "epoch 51, iter 20250, avg. loss 66.21, avg. ppl 16.61 cum. examples 3998, speed 1314.90 words/sec, time elapsed 5389.61 sec\n",
            "epoch 51, iter 20260, avg. loss 60.73, avg. ppl 16.17 cum. examples 4158, speed 1208.21 words/sec, time elapsed 5392.50 sec\n",
            "epoch 51, iter 20270, avg. loss 67.35, avg. ppl 18.17 cum. examples 4318, speed 1229.07 words/sec, time elapsed 5395.53 sec\n",
            "epoch 51, iter 20280, avg. loss 57.47, avg. ppl 14.14 cum. examples 4478, speed 1206.76 words/sec, time elapsed 5398.40 sec\n",
            "epoch 51, iter 20290, avg. loss 60.84, avg. ppl 15.22 cum. examples 4638, speed 1409.56 words/sec, time elapsed 5400.94 sec\n",
            "epoch 51, iter 20300, avg. loss 63.66, avg. ppl 16.16 cum. examples 4798, speed 1194.90 words/sec, time elapsed 5404.00 sec\n",
            "epoch 51, iter 20310, avg. loss 55.51, avg. ppl 13.30 cum. examples 4958, speed 1505.34 words/sec, time elapsed 5406.28 sec\n",
            "epoch 51, iter 20320, avg. loss 66.75, avg. ppl 15.84 cum. examples 5118, speed 1199.62 words/sec, time elapsed 5409.50 sec\n",
            "epoch 51, iter 20330, avg. loss 67.03, avg. ppl 17.05 cum. examples 5278, speed 1200.02 words/sec, time elapsed 5412.66 sec\n",
            "epoch 51, iter 20340, avg. loss 60.29, avg. ppl 15.91 cum. examples 5438, speed 1340.84 words/sec, time elapsed 5415.26 sec\n",
            "epoch 51, iter 20350, avg. loss 50.91, avg. ppl 12.58 cum. examples 5598, speed 1538.03 words/sec, time elapsed 5417.35 sec\n",
            "epoch 51, iter 20360, avg. loss 56.45, avg. ppl 13.98 cum. examples 5758, speed 1574.82 words/sec, time elapsed 5419.52 sec\n",
            "epoch 51, iter 20370, avg. loss 63.58, avg. ppl 16.13 cum. examples 5918, speed 1141.74 words/sec, time elapsed 5422.73 sec\n",
            "epoch 51, iter 20380, avg. loss 51.64, avg. ppl 12.94 cum. examples 6078, speed 1446.92 words/sec, time elapsed 5424.96 sec\n",
            "epoch 51, iter 20390, avg. loss 56.94, avg. ppl 14.52 cum. examples 6238, speed 1382.79 words/sec, time elapsed 5427.42 sec\n",
            "epoch 51, iter 20400, avg. loss 58.05, avg. ppl 13.99 cum. examples 6398, speed 1431.00 words/sec, time elapsed 5429.88 sec\n",
            "epoch 51, iter 20410, avg. loss 54.41, avg. ppl 13.93 cum. examples 6558, speed 1392.14 words/sec, time elapsed 5432.25 sec\n",
            "epoch 51, iter 20420, avg. loss 52.17, avg. ppl 12.87 cum. examples 6718, speed 1307.38 words/sec, time elapsed 5434.75 sec\n",
            "epoch 51, iter 20430, avg. loss 63.53, avg. ppl 16.48 cum. examples 6878, speed 1478.85 words/sec, time elapsed 5437.20 sec\n",
            "epoch 51, iter 20440, avg. loss 58.23, avg. ppl 15.38 cum. examples 7038, speed 1472.71 words/sec, time elapsed 5439.52 sec\n",
            "epoch 51, iter 20450, avg. loss 56.86, avg. ppl 14.36 cum. examples 7198, speed 1280.00 words/sec, time elapsed 5442.19 sec\n",
            "epoch 52, iter 20460, avg. loss 58.92, avg. ppl 13.69 cum. examples 7356, speed 1435.86 words/sec, time elapsed 5444.66 sec\n",
            "epoch 52, iter 20470, avg. loss 54.69, avg. ppl 12.87 cum. examples 7516, speed 1334.68 words/sec, time elapsed 5447.23 sec\n",
            "epoch 52, iter 20480, avg. loss 49.30, avg. ppl 11.39 cum. examples 7676, speed 1579.73 words/sec, time elapsed 5449.28 sec\n",
            "epoch 52, iter 20490, avg. loss 51.70, avg. ppl 12.20 cum. examples 7836, speed 1568.85 words/sec, time elapsed 5451.39 sec\n",
            "epoch 52, iter 20500, avg. loss 50.81, avg. ppl 11.69 cum. examples 7996, speed 1430.73 words/sec, time elapsed 5453.70 sec\n",
            "epoch 52, iter 20510, avg. loss 59.30, avg. ppl 13.75 cum. examples 8156, speed 1484.53 words/sec, time elapsed 5456.14 sec\n",
            "epoch 52, iter 20520, avg. loss 63.84, avg. ppl 14.55 cum. examples 8316, speed 1443.96 words/sec, time elapsed 5458.78 sec\n",
            "epoch 52, iter 20530, avg. loss 62.22, avg. ppl 14.59 cum. examples 8476, speed 1322.70 words/sec, time elapsed 5461.59 sec\n",
            "epoch 52, iter 20540, avg. loss 58.58, avg. ppl 13.83 cum. examples 8636, speed 1349.82 words/sec, time elapsed 5464.23 sec\n",
            "epoch 52, iter 20550, avg. loss 54.18, avg. ppl 12.72 cum. examples 8796, speed 1529.79 words/sec, time elapsed 5466.46 sec\n",
            "epoch 52, iter 20560, avg. loss 56.40, avg. ppl 13.08 cum. examples 8956, speed 1435.51 words/sec, time elapsed 5468.91 sec\n",
            "epoch 52, iter 20570, avg. loss 55.64, avg. ppl 12.87 cum. examples 9116, speed 1506.94 words/sec, time elapsed 5471.22 sec\n",
            "epoch 52, iter 20580, avg. loss 51.65, avg. ppl 11.68 cum. examples 9276, speed 1271.45 words/sec, time elapsed 5473.86 sec\n",
            "epoch 52, iter 20590, avg. loss 72.53, avg. ppl 18.54 cum. examples 9436, speed 1294.17 words/sec, time elapsed 5476.93 sec\n",
            "epoch 52, iter 20600, avg. loss 54.33, avg. ppl 12.96 cum. examples 9596, speed 1412.50 words/sec, time elapsed 5479.34 sec\n",
            "epoch 52, iter 20610, avg. loss 60.66, avg. ppl 13.94 cum. examples 9756, speed 1639.24 words/sec, time elapsed 5481.58 sec\n",
            "epoch 52, iter 20620, avg. loss 59.95, avg. ppl 13.79 cum. examples 9916, speed 1390.64 words/sec, time elapsed 5484.21 sec\n",
            "epoch 52, iter 20630, avg. loss 51.70, avg. ppl 12.24 cum. examples 10076, speed 1579.32 words/sec, time elapsed 5486.30 sec\n",
            "epoch 52, iter 20640, avg. loss 57.94, avg. ppl 13.88 cum. examples 10236, speed 1131.42 words/sec, time elapsed 5489.42 sec\n",
            "epoch 52, iter 20650, avg. loss 61.58, avg. ppl 13.81 cum. examples 10396, speed 1110.67 words/sec, time elapsed 5492.80 sec\n",
            "epoch 52, iter 20660, avg. loss 58.00, avg. ppl 13.60 cum. examples 10556, speed 1318.19 words/sec, time elapsed 5495.49 sec\n",
            "epoch 52, iter 20670, avg. loss 54.08, avg. ppl 13.16 cum. examples 10716, speed 1562.19 words/sec, time elapsed 5497.64 sec\n",
            "epoch 52, iter 20680, avg. loss 66.04, avg. ppl 15.88 cum. examples 10876, speed 1245.78 words/sec, time elapsed 5500.71 sec\n",
            "epoch 52, iter 20690, avg. loss 56.37, avg. ppl 12.78 cum. examples 11036, speed 1362.27 words/sec, time elapsed 5503.31 sec\n",
            "epoch 52, iter 20700, avg. loss 50.32, avg. ppl 11.62 cum. examples 11196, speed 1489.90 words/sec, time elapsed 5505.51 sec\n",
            "epoch 52, iter 20710, avg. loss 57.96, avg. ppl 13.71 cum. examples 11356, speed 1302.57 words/sec, time elapsed 5508.23 sec\n",
            "epoch 52, iter 20720, avg. loss 67.25, avg. ppl 15.79 cum. examples 11516, speed 1293.13 words/sec, time elapsed 5511.25 sec\n",
            "epoch 52, iter 20730, avg. loss 54.44, avg. ppl 12.96 cum. examples 11676, speed 1411.15 words/sec, time elapsed 5513.66 sec\n",
            "epoch 52, iter 20740, avg. loss 54.02, avg. ppl 12.77 cum. examples 11836, speed 1367.58 words/sec, time elapsed 5516.14 sec\n",
            "epoch 52, iter 20750, avg. loss 64.64, avg. ppl 14.73 cum. examples 11996, speed 1418.46 words/sec, time elapsed 5518.85 sec\n",
            "epoch 52, iter 20760, avg. loss 64.30, avg. ppl 15.31 cum. examples 12156, speed 1327.94 words/sec, time elapsed 5521.69 sec\n",
            "epoch 52, iter 20770, avg. loss 65.23, avg. ppl 16.00 cum. examples 12316, speed 1321.38 words/sec, time elapsed 5524.54 sec\n",
            "epoch 52, iter 20780, avg. loss 52.89, avg. ppl 13.55 cum. examples 12476, speed 1040.04 words/sec, time elapsed 5527.66 sec\n",
            "epoch 52, iter 20790, avg. loss 57.69, avg. ppl 14.29 cum. examples 12636, speed 1307.78 words/sec, time elapsed 5530.31 sec\n",
            "epoch 52, iter 20800, avg. loss 62.43, avg. ppl 15.55 cum. examples 12796, speed 1152.35 words/sec, time elapsed 5533.47 sec\n",
            "epoch 52, iter 20800, cum. loss 58.77, cum. ppl 14.41 cum. examples 12796\n",
            "begin validation ...\n",
            "validation: iter 20800, dev. ppl 76.342537\n",
            "hit patience 4\n",
            "epoch 52, iter 20810, avg. loss 65.17, avg. ppl 15.76 cum. examples 160, speed 450.92 words/sec, time elapsed 5541.86 sec\n",
            "epoch 52, iter 20820, avg. loss 55.57, avg. ppl 13.34 cum. examples 320, speed 1379.33 words/sec, time elapsed 5544.35 sec\n",
            "epoch 52, iter 20830, avg. loss 47.17, avg. ppl 11.15 cum. examples 480, speed 1559.64 words/sec, time elapsed 5546.35 sec\n",
            "epoch 52, iter 20840, avg. loss 60.67, avg. ppl 14.03 cum. examples 640, speed 1499.69 words/sec, time elapsed 5548.80 sec\n",
            "epoch 52, iter 20850, avg. loss 52.69, avg. ppl 12.87 cum. examples 800, speed 1510.17 words/sec, time elapsed 5550.99 sec\n",
            "epoch 53, iter 20860, avg. loss 57.88, avg. ppl 13.25 cum. examples 958, speed 1351.74 words/sec, time elapsed 5553.61 sec\n",
            "epoch 53, iter 20870, avg. loss 53.50, avg. ppl 11.65 cum. examples 1118, speed 1305.42 words/sec, time elapsed 5556.28 sec\n",
            "epoch 53, iter 20880, avg. loss 55.82, avg. ppl 12.43 cum. examples 1278, speed 1454.85 words/sec, time elapsed 5558.71 sec\n",
            "epoch 53, iter 20890, avg. loss 55.03, avg. ppl 12.42 cum. examples 1438, speed 1496.81 words/sec, time elapsed 5561.05 sec\n",
            "epoch 53, iter 20900, avg. loss 52.52, avg. ppl 11.98 cum. examples 1598, speed 1317.11 words/sec, time elapsed 5563.62 sec\n",
            "epoch 53, iter 20910, avg. loss 57.29, avg. ppl 12.37 cum. examples 1758, speed 1380.44 words/sec, time elapsed 5566.26 sec\n",
            "epoch 53, iter 20920, avg. loss 46.38, avg. ppl 10.08 cum. examples 1918, speed 1405.38 words/sec, time elapsed 5568.54 sec\n",
            "epoch 53, iter 20930, avg. loss 55.71, avg. ppl 12.62 cum. examples 2078, speed 1005.00 words/sec, time elapsed 5572.04 sec\n",
            "epoch 53, iter 20940, avg. loss 64.79, avg. ppl 14.29 cum. examples 2238, speed 1239.80 words/sec, time elapsed 5575.19 sec\n",
            "epoch 53, iter 20950, avg. loss 62.36, avg. ppl 13.40 cum. examples 2398, speed 1330.20 words/sec, time elapsed 5578.08 sec\n",
            "epoch 53, iter 20960, avg. loss 70.04, avg. ppl 16.79 cum. examples 2558, speed 1408.70 words/sec, time elapsed 5580.90 sec\n",
            "epoch 53, iter 20970, avg. loss 51.38, avg. ppl 11.52 cum. examples 2718, speed 1432.66 words/sec, time elapsed 5583.24 sec\n",
            "epoch 53, iter 20980, avg. loss 56.94, avg. ppl 13.48 cum. examples 2878, speed 1253.82 words/sec, time elapsed 5586.04 sec\n",
            "epoch 53, iter 20990, avg. loss 60.02, avg. ppl 13.59 cum. examples 3038, speed 1321.15 words/sec, time elapsed 5588.82 sec\n",
            "epoch 53, iter 21000, avg. loss 47.37, avg. ppl 10.34 cum. examples 3198, speed 1475.55 words/sec, time elapsed 5591.02 sec\n",
            "epoch 53, iter 21010, avg. loss 53.70, avg. ppl 11.82 cum. examples 3358, speed 1464.63 words/sec, time elapsed 5593.40 sec\n",
            "epoch 53, iter 21020, avg. loss 50.39, avg. ppl 11.02 cum. examples 3518, speed 1134.41 words/sec, time elapsed 5596.36 sec\n",
            "epoch 53, iter 21030, avg. loss 59.13, avg. ppl 13.04 cum. examples 3678, speed 1463.34 words/sec, time elapsed 5598.88 sec\n",
            "epoch 53, iter 21040, avg. loss 58.89, avg. ppl 13.20 cum. examples 3838, speed 1448.79 words/sec, time elapsed 5601.40 sec\n",
            "epoch 53, iter 21050, avg. loss 57.97, avg. ppl 12.41 cum. examples 3998, speed 1320.12 words/sec, time elapsed 5604.19 sec\n",
            "epoch 53, iter 21060, avg. loss 59.43, avg. ppl 13.35 cum. examples 4158, speed 1364.41 words/sec, time elapsed 5606.88 sec\n",
            "epoch 53, iter 21070, avg. loss 50.11, avg. ppl 11.66 cum. examples 4318, speed 1501.94 words/sec, time elapsed 5609.05 sec\n",
            "epoch 53, iter 21080, avg. loss 62.81, avg. ppl 14.37 cum. examples 4478, speed 1253.47 words/sec, time elapsed 5612.06 sec\n",
            "epoch 53, iter 21090, avg. loss 57.78, avg. ppl 12.81 cum. examples 4638, speed 1428.76 words/sec, time elapsed 5614.60 sec\n",
            "epoch 53, iter 21100, avg. loss 51.08, avg. ppl 11.24 cum. examples 4798, speed 1490.09 words/sec, time elapsed 5616.86 sec\n",
            "epoch 53, iter 21110, avg. loss 55.12, avg. ppl 12.56 cum. examples 4958, speed 1253.30 words/sec, time elapsed 5619.64 sec\n",
            "epoch 53, iter 21120, avg. loss 58.07, avg. ppl 12.65 cum. examples 5118, speed 1386.37 words/sec, time elapsed 5622.28 sec\n",
            "epoch 53, iter 21130, avg. loss 51.12, avg. ppl 11.42 cum. examples 5278, speed 1334.79 words/sec, time elapsed 5624.80 sec\n",
            "epoch 53, iter 21140, avg. loss 56.86, avg. ppl 13.10 cum. examples 5438, speed 1475.93 words/sec, time elapsed 5627.20 sec\n",
            "epoch 53, iter 21150, avg. loss 53.74, avg. ppl 11.20 cum. examples 5598, speed 1396.33 words/sec, time elapsed 5629.74 sec\n",
            "epoch 53, iter 21160, avg. loss 46.11, avg. ppl 10.21 cum. examples 5758, speed 1629.89 words/sec, time elapsed 5631.69 sec\n",
            "epoch 53, iter 21170, avg. loss 57.19, avg. ppl 13.57 cum. examples 5918, speed 1254.09 words/sec, time elapsed 5634.49 sec\n",
            "epoch 53, iter 21180, avg. loss 52.91, avg. ppl 11.44 cum. examples 6078, speed 1409.61 words/sec, time elapsed 5636.95 sec\n",
            "epoch 53, iter 21190, avg. loss 59.62, avg. ppl 13.91 cum. examples 6238, speed 1335.56 words/sec, time elapsed 5639.67 sec\n",
            "epoch 53, iter 21200, avg. loss 54.95, avg. ppl 12.42 cum. examples 6398, speed 1430.03 words/sec, time elapsed 5642.11 sec\n",
            "epoch 53, iter 21210, avg. loss 61.50, avg. ppl 13.58 cum. examples 6558, speed 1448.53 words/sec, time elapsed 5644.71 sec\n",
            "epoch 53, iter 21220, avg. loss 50.23, avg. ppl 11.13 cum. examples 6718, speed 1434.21 words/sec, time elapsed 5647.04 sec\n",
            "epoch 53, iter 21230, avg. loss 57.10, avg. ppl 12.63 cum. examples 6878, speed 1317.41 words/sec, time elapsed 5649.77 sec\n",
            "epoch 53, iter 21240, avg. loss 57.08, avg. ppl 12.34 cum. examples 7038, speed 1397.26 words/sec, time elapsed 5652.37 sec\n",
            "epoch 53, iter 21250, avg. loss 51.38, avg. ppl 11.61 cum. examples 7198, speed 1375.47 words/sec, time elapsed 5654.81 sec\n",
            "epoch 54, iter 21260, avg. loss 54.97, avg. ppl 11.31 cum. examples 7356, speed 1215.03 words/sec, time elapsed 5657.76 sec\n",
            "epoch 54, iter 21270, avg. loss 46.60, avg. ppl 9.71 cum. examples 7516, speed 1444.60 words/sec, time elapsed 5660.03 sec\n",
            "epoch 54, iter 21280, avg. loss 53.93, avg. ppl 11.66 cum. examples 7676, speed 1436.15 words/sec, time elapsed 5662.48 sec\n",
            "epoch 54, iter 21290, avg. loss 47.45, avg. ppl 10.24 cum. examples 7836, speed 1398.94 words/sec, time elapsed 5664.81 sec\n",
            "epoch 54, iter 21300, avg. loss 56.78, avg. ppl 12.39 cum. examples 7996, speed 1360.06 words/sec, time elapsed 5667.46 sec\n",
            "epoch 54, iter 21310, avg. loss 47.94, avg. ppl 10.06 cum. examples 8156, speed 1487.42 words/sec, time elapsed 5669.70 sec\n",
            "epoch 54, iter 21320, avg. loss 53.69, avg. ppl 11.09 cum. examples 8316, speed 1119.40 words/sec, time elapsed 5672.89 sec\n",
            "epoch 54, iter 21330, avg. loss 46.26, avg. ppl 9.44 cum. examples 8476, speed 1450.95 words/sec, time elapsed 5675.16 sec\n",
            "epoch 54, iter 21340, avg. loss 55.94, avg. ppl 11.72 cum. examples 8636, speed 1632.46 words/sec, time elapsed 5677.39 sec\n",
            "epoch 54, iter 21350, avg. loss 48.96, avg. ppl 10.91 cum. examples 8796, speed 1356.26 words/sec, time elapsed 5679.80 sec\n",
            "epoch 54, iter 21360, avg. loss 63.74, avg. ppl 13.23 cum. examples 8956, speed 1252.21 words/sec, time elapsed 5682.96 sec\n",
            "epoch 54, iter 21370, avg. loss 53.16, avg. ppl 11.34 cum. examples 9116, speed 1541.14 words/sec, time elapsed 5685.23 sec\n",
            "epoch 54, iter 21380, avg. loss 50.61, avg. ppl 11.25 cum. examples 9276, speed 1395.50 words/sec, time elapsed 5687.63 sec\n",
            "epoch 54, iter 21390, avg. loss 55.89, avg. ppl 11.83 cum. examples 9436, speed 1551.82 words/sec, time elapsed 5689.96 sec\n",
            "epoch 54, iter 21400, avg. loss 55.60, avg. ppl 11.79 cum. examples 9596, speed 1493.67 words/sec, time elapsed 5692.38 sec\n",
            "epoch 54, iter 21410, avg. loss 57.82, avg. ppl 12.93 cum. examples 9756, speed 1242.50 words/sec, time elapsed 5695.28 sec\n",
            "epoch 54, iter 21420, avg. loss 57.35, avg. ppl 12.04 cum. examples 9916, speed 1392.42 words/sec, time elapsed 5697.93 sec\n",
            "epoch 54, iter 21430, avg. loss 61.02, avg. ppl 12.93 cum. examples 10076, speed 1184.73 words/sec, time elapsed 5701.15 sec\n",
            "epoch 54, iter 21440, avg. loss 52.09, avg. ppl 10.93 cum. examples 10236, speed 1568.16 words/sec, time elapsed 5703.37 sec\n",
            "epoch 54, iter 21450, avg. loss 52.10, avg. ppl 10.79 cum. examples 10396, speed 1275.47 words/sec, time elapsed 5706.12 sec\n",
            "epoch 54, iter 21460, avg. loss 57.84, avg. ppl 11.52 cum. examples 10556, speed 1307.44 words/sec, time elapsed 5709.02 sec\n",
            "epoch 54, iter 21470, avg. loss 54.22, avg. ppl 11.68 cum. examples 10716, speed 1283.78 words/sec, time elapsed 5711.77 sec\n",
            "epoch 54, iter 21480, avg. loss 56.35, avg. ppl 11.50 cum. examples 10876, speed 1382.93 words/sec, time elapsed 5714.44 sec\n",
            "epoch 54, iter 21490, avg. loss 57.65, avg. ppl 12.37 cum. examples 11036, speed 1453.97 words/sec, time elapsed 5716.96 sec\n",
            "epoch 54, iter 21500, avg. loss 54.54, avg. ppl 11.81 cum. examples 11196, speed 1305.42 words/sec, time elapsed 5719.67 sec\n",
            "epoch 54, iter 21510, avg. loss 55.28, avg. ppl 11.49 cum. examples 11356, speed 1469.20 words/sec, time elapsed 5722.13 sec\n",
            "epoch 54, iter 21520, avg. loss 47.87, avg. ppl 10.30 cum. examples 11516, speed 1697.83 words/sec, time elapsed 5724.07 sec\n",
            "epoch 54, iter 21530, avg. loss 57.09, avg. ppl 11.64 cum. examples 11676, speed 1302.48 words/sec, time elapsed 5726.92 sec\n",
            "epoch 54, iter 21540, avg. loss 50.49, avg. ppl 10.27 cum. examples 11836, speed 1395.99 words/sec, time elapsed 5729.41 sec\n",
            "epoch 54, iter 21550, avg. loss 49.11, avg. ppl 10.62 cum. examples 11996, speed 1168.40 words/sec, time elapsed 5732.25 sec\n",
            "epoch 54, iter 21560, avg. loss 56.51, avg. ppl 11.71 cum. examples 12156, speed 1316.26 words/sec, time elapsed 5735.05 sec\n",
            "epoch 54, iter 21570, avg. loss 55.59, avg. ppl 11.73 cum. examples 12316, speed 1238.46 words/sec, time elapsed 5737.96 sec\n",
            "epoch 54, iter 21580, avg. loss 54.23, avg. ppl 11.59 cum. examples 12476, speed 1289.30 words/sec, time elapsed 5740.71 sec\n",
            "epoch 54, iter 21590, avg. loss 51.27, avg. ppl 10.75 cum. examples 12636, speed 1462.47 words/sec, time elapsed 5743.07 sec\n",
            "epoch 54, iter 21600, avg. loss 54.54, avg. ppl 11.25 cum. examples 12796, speed 1443.29 words/sec, time elapsed 5745.57 sec\n",
            "epoch 54, iter 21600, cum. loss 54.96, cum. ppl 12.03 cum. examples 12796\n",
            "begin validation ...\n",
            "validation: iter 21600, dev. ppl 84.549717\n",
            "hit patience 5\n",
            "hit #5 trial\n",
            "early stop!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "GKGy4mT3OclR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 156
        },
        "outputId": "fa42be9e-a363-40dd-9254-2745c10f500b"
      },
      "cell_type": "code",
      "source": [
        "!bash decodeMLE.sh en-az"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "source az\n",
            "target en\n",
            "load model from results/en-az/model.bin\n",
            "Decoding:   0% 0/998 [00:00<?, ?it/s]/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py:1320: UserWarning: nn.functional.tanh is deprecated. Use torch.tanh instead.\n",
            "  warnings.warn(\"nn.functional.tanh is deprecated. Use torch.tanh instead.\")\n",
            "Decoding: 100% 998/998 [02:58<00:00,  4.62it/s]\n",
            "BLEU = 2.92, 22.2/4.7/1.5/0.5 (BP=1.000, ratio=1.170, hyp_len=25271, ref_len=21604)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "dvSxBXWRSMRw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 156
        },
        "outputId": "0f506968-0c52-46f7-9a3b-2f7f9ae6cf61"
      },
      "cell_type": "code",
      "source": [
        "!bash decodeOT.sh en-az"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "source az\n",
            "target en\n",
            "load model from results/en-az/model.bin\n",
            "Decoding:   0% 0/998 [00:00<?, ?it/s]/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py:1320: UserWarning: nn.functional.tanh is deprecated. Use torch.tanh instead.\n",
            "  warnings.warn(\"nn.functional.tanh is deprecated. Use torch.tanh instead.\")\n",
            "Decoding: 100% 998/998 [03:45<00:00,  3.73it/s]\n",
            "BLEU = 2.92, 22.2/4.7/1.5/0.5 (BP=1.000, ratio=1.170, hyp_len=25271, ref_len=21604)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "LwOf20fsiMli",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}