16. RNN_practice.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b2dc34a9",
   "metadata": {},
   "source": [
    "## One-Hot-Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "97949827",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [1. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [1. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [1. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 1. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 1. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 1. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 1. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 1. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 1. 0. 0. 0.]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-12-10 09:18:03.954498: I tensorflow/core/platform/cpu_feature_guard.cc:145] This TensorFlow binary is optimized with Intel(R) MKL-DNN to use the following CPU instructions in performance critical operations:  SSE4.1 SSE4.2 AVX\n",
      "To enable them in non-MKL-DNN operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2021-12-10 09:18:03.955052: I tensorflow/core/common_runtime/process_util.cc:115] Creating new thread pool with default inter op setting: 8. Tune using inter_op_parallelism_threads for best performance.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "vocab = {      # 사용할 단어 사전 정의\n",
    "    \"i\": 0,\n",
    "    \"need\": 1,\n",
    "    \"some\": 2,\n",
    "    \"more\": 3,\n",
    "    \"coffee\": 4,\n",
    "    \"cake\": 5,\n",
    "    \"cat\": 6,\n",
    "    \"dog\": 7\n",
    "}\n",
    "\n",
    "sentence = \"i i i i need some more coffee coffee coffee\"\n",
    "# 위 sentence\n",
    "_input = [vocab[w] for w in sentence.split()]  # [0, 0, 0, 0, 1, 2, 3, 4, 4, 4]\n",
    "\n",
    "vocab_size = len(vocab)   # 8\n",
    "\n",
    "one_hot = tf.one_hot(_input, vocab_size)\n",
    "print(one_hot.numpy())    # 원-핫 인코딩 벡터를 출력해 봅시다."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d1a5834a",
   "metadata": {},
   "source": [
    "## 단어의 분산 표현"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "dd8f061e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Linear Weight\n",
      "[[-0.73741895  0.16297466]\n",
      " [-0.7259325  -0.69949377]\n",
      " [ 0.14495468 -0.1279521 ]\n",
      " [-0.45607862 -0.62413394]\n",
      " [-0.10111594  0.15959525]\n",
      " [-0.75735676 -0.67651296]\n",
      " [ 0.24570465  0.3908615 ]\n",
      " [ 0.15521061  0.6642498 ]]\n",
      "\n",
      "One-Hot Linear Result\n",
      "[[-0.73741895  0.16297466]\n",
      " [-0.73741895  0.16297466]\n",
      " [-0.73741895  0.16297466]\n",
      " [-0.73741895  0.16297466]\n",
      " [-0.7259325  -0.69949377]\n",
      " [ 0.14495468 -0.1279521 ]\n",
      " [-0.45607862 -0.62413394]\n",
      " [-0.10111594  0.15959525]\n",
      " [-0.10111594  0.15959525]\n",
      " [-0.10111594  0.15959525]]\n"
     ]
    }
   ],
   "source": [
    "distribution_size = 2   # 보기 좋게 2차원으로 분산 표현하도록 하죠!\n",
    "linear = tf.keras.layers.Dense(units=distribution_size, use_bias=False)\n",
    "one_hot_linear = linear(one_hot)\n",
    "\n",
    "print(\"Linear Weight\")\n",
    "print(linear.weights[0].numpy())\n",
    "\n",
    "print(\"\\nOne-Hot Linear Result\")\n",
    "print(one_hot_linear.numpy())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "af278bb4",
   "metadata": {},
   "source": [
    "## embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c50fe859",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Embedding을 진행할 문장: (1, 3)\n",
      "Embedding된 문장: (1, 3, 100)\n",
      "Embedding Layer의 Weight 형태: (64, 100)\n"
     ]
    }
   ],
   "source": [
    "some_words = tf.constant([[3, 57, 35]])\n",
    "# 3번 단어 / 57번 단어 / 35번 단어로 이루어진 한 문장입니다.\n",
    "\n",
    "print(\"Embedding을 진행할 문장:\", some_words.shape)\n",
    "embedding_layer = tf.keras.layers.Embedding(input_dim=64, output_dim=100)\n",
    "# 총 64개의 단어를 포함한 Embedding 레이어를 선언할 것이고,\n",
    "# 각 단어는 100차원으로 분산표현 할 것입니다.\n",
    "\n",
    "print(\"Embedding된 문장:\", embedding_layer(some_words).shape)\n",
    "print(\"Embedding Layer의 Weight 형태:\", embedding_layer.weights[0].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "bc99598b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=23, shape=(1, 3), dtype=int32, numpy=array([[ 3, 57, 35]], dtype=int32)>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.keras.layers.Embedding()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "88542f85",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RNN에 입력할 문장: What time is it ?\n",
      "Embedding을 위해 단어 매핑: [[2 3 0 1 4]]\n",
      "입력 문장 데이터 형태: (1, 5)\n",
      "\n",
      "Embedding 결과: (1, 5, 100)\n",
      "Embedding Layer의 Weight 형태: (5, 100)\n",
      "\n",
      "RNN 결과 (모든 Step Output): (1, 5, 64)\n",
      "RNN Layer의 Weight 형태: (100, 64)\n",
      "\n",
      "RNN 결과 (최종 Step Output): (1, 64)\n",
      "RNN Layer의 Weight 형태: (100, 64)\n"
     ]
    }
   ],
   "source": [
    "sentence = \"What time is it ?\"\n",
    "dic = {\n",
    "    \"is\": 0,\n",
    "    \"it\": 1,\n",
    "    \"What\": 2,\n",
    "    \"time\": 3,\n",
    "    \"?\": 4\n",
    "}\n",
    "\n",
    "print(\"RNN에 입력할 문장:\", sentence)\n",
    "\n",
    "sentence_tensor = tf.constant([[dic[word] for word in sentence.split()]])\n",
    "\n",
    "print(\"Embedding을 위해 단어 매핑:\", sentence_tensor.numpy())\n",
    "print(\"입력 문장 데이터 형태:\", sentence_tensor.shape)\n",
    "\n",
    "embedding_layer = tf.keras.layers.Embedding(input_dim=len(dic), output_dim=100)\n",
    "emb_out = embedding_layer(sentence_tensor)\n",
    "\n",
    "print(\"\\nEmbedding 결과:\", emb_out.shape)\n",
    "print(\"Embedding Layer의 Weight 형태:\", embedding_layer.weights[0].shape)\n",
    "\n",
    "rnn_seq_layer = \\\n",
    "tf.keras.layers.SimpleRNN(units=64, return_sequences=True, use_bias=False)\n",
    "rnn_seq_out = rnn_seq_layer(emb_out)\n",
    "\n",
    "print(\"\\nRNN 결과 (모든 Step Output):\", rnn_seq_out.shape)\n",
    "print(\"RNN Layer의 Weight 형태:\", rnn_seq_layer.weights[0].shape)\n",
    "\n",
    "rnn_fin_layer = tf.keras.layers.SimpleRNN(units=64, use_bias=False)\n",
    "rnn_fin_out = rnn_fin_layer(emb_out)\n",
    "\n",
    "print(\"\\nRNN 결과 (최종 Step Output):\", rnn_fin_out.shape)\n",
    "print(\"RNN Layer의 Weight 형태:\", rnn_fin_layer.weights[0].shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0887ac2b",
   "metadata": {},
   "source": [
    "## LSTM 사용"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "cd57d90b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "LSTM 결과 (모든 Step Output): (1, 5, 64)\n",
      "LSTM Layer의 Weight 형태: (100, 256)\n",
      "\n",
      "LSTM 결과 (최종 Step Output): (1, 64)\n",
      "LSTM Layer의 Weight 형태: (100, 256)\n"
     ]
    }
   ],
   "source": [
    "# LSTM으로 바꿔보기\n",
    "# use_bias만 True로 바꾸면 됨\n",
    "\n",
    "lstm_seq_layer = tf.keras.layers.LSTM(units=64, return_sequences=True, use_bias=True) ## Your Code ##\n",
    "lstm_seq_out = lstm_seq_layer(emb_out)\n",
    "\n",
    "print(\"\\nLSTM 결과 (모든 Step Output):\", lstm_seq_out.shape)\n",
    "print(\"LSTM Layer의 Weight 형태:\", lstm_seq_layer.weights[0].shape)\n",
    "\n",
    "lstm_fin_layer = tf.keras.layers.LSTM(units=64, use_bias=True)  ## Your Code ##\n",
    "lstm_fin_out = lstm_fin_layer(emb_out)\n",
    "\n",
    "print(\"\\nLSTM 결과 (최종 Step Output):\", lstm_fin_out.shape)\n",
    "print(\"LSTM Layer의 Weight 형태:\", lstm_fin_layer.weights[0].shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bb2e465e",
   "metadata": {},
   "source": [
    "## GRU 사용"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "c5f683c1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RNN에 입력할 문장: What time is it ?\n",
      "Embedding을 위해 단어 매핑: [[2 3 0 1 4]]\n",
      "입력 문장 데이터 형태: (1, 5)\n",
      "\n",
      "Embedding 결과: (1, 5, 100)\n",
      "Embedding Layer의 Weight 형태: (5, 100)\n",
      "\n",
      "GRU 결과 (모든 Step Output): (1, 5, 64)\n",
      "GRU Layer의 Weight 형태: (100, 192)\n",
      "\n",
      "GRU 결과 (최종 Step Output): (1, 64)\n",
      "GRU Layer의 Weight 형태: (100, 192)\n"
     ]
    }
   ],
   "source": [
    "sentence = \"What time is it ?\"\n",
    "dic = {\n",
    "    \"is\": 0,\n",
    "    \"it\": 1,\n",
    "    \"What\": 2,\n",
    "    \"time\": 3,\n",
    "    \"?\": 4\n",
    "}\n",
    "\n",
    "print(\"RNN에 입력할 문장:\", sentence)\n",
    "\n",
    "sentence_tensor = tf.constant([[dic[word] for word in sentence.split()]])\n",
    "\n",
    "print(\"Embedding을 위해 단어 매핑:\", sentence_tensor.numpy())\n",
    "print(\"입력 문장 데이터 형태:\", sentence_tensor.shape)\n",
    "\n",
    "embedding_layer = tf.keras.layers.Embedding(input_dim=len(dic), output_dim=100) ### YOUR CODE ###\n",
    "emb_out = embedding_layer(sentence_tensor)\n",
    "\n",
    "print(\"\\nEmbedding 결과:\", emb_out.shape)\n",
    "print(\"Embedding Layer의 Weight 형태:\", embedding_layer.weights[0].shape)\n",
    "\n",
    "gru_seq_layer = tf.keras.layers.GRU(units=64, return_sequences=True, use_bias=True) ### YOUR CODE ###\n",
    "gru_seq_out = gru_seq_layer(emb_out)\n",
    "\n",
    "print(\"\\nGRU 결과 (모든 Step Output):\", gru_seq_out.shape)\n",
    "print(\"GRU Layer의 Weight 형태:\", gru_seq_layer.weights[0].shape)\n",
    "\n",
    "gru_fin_layer = tf.keras.layers.GRU(units=64, use_bias=True) ### YOUR CODE ###\n",
    "gru_fin_out = gru_fin_layer(emb_out)\n",
    "\n",
    "print(\"\\nGRU 결과 (최종 Step Output):\", gru_fin_out.shape)\n",
    "print(\"GRU Layer의 Weight 형태:\", gru_fin_layer.weights[0].shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9058291e",
   "metadata": {},
   "source": [
    "## Bidirectional 사용"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "d94a0493",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "입력 문장 데이터 형태: (1, 5, 100)\n",
      "Bidirectional RNN 결과 (최종 Step Output): (1, 5, 128)\n"
     ]
    }
   ],
   "source": [
    "embedding_layer = tf.keras.layers.Embedding(input_dim=len(dic), output_dim=100)\n",
    "emb_out = embedding_layer(sentence_tensor)\n",
    "\n",
    "print(\"입력 문장 데이터 형태:\", emb_out.shape)\n",
    "\n",
    "bi_rnn = \\\n",
    "tf.keras.layers.Bidirectional(\n",
    "    tf.keras.layers.SimpleRNN(units=64, use_bias=False, return_sequences=True)\n",
    ")\n",
    "bi_out = bi_rnn(emb_out)\n",
    "\n",
    "print(\"Bidirectional RNN 결과 (최종 Step Output):\", bi_out.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "560b88a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "simple = tf.keras.layers.SimpleRNN(units=64, use_bias=False, return_sequences=True)\n",
    "simple(emb_out).shape\n",
    "\n",
    "bi = tf.keras.layers.Bidirectional(simple)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "bd9cb24e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.layers.wrappers.Bidirectional at 0x7f83efc68a10>"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "48b16248",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([1, 5, 128])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bi(emb_out).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b7647bc",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}