0.expression-download/2B.process-target-data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Process TARGET PanCancer Data\n",
    "\n",
    "Retrieve the downloaded expression data, update gene identifiers to entrez, and curate sample IDs. The script will also identify a balanced hold-out test set to compare projection performance into learned latent spaces across algorithms."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(1234)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read Phenotype Information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5958, 7)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sample_id</th>\n",
       "      <th>primary_disease_code</th>\n",
       "      <th>_primary_disease</th>\n",
       "      <th>sample_type_code</th>\n",
       "      <th>_sample_type</th>\n",
       "      <th>_PATIENT</th>\n",
       "      <th>_cohort</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TARGET-00-NAAEMA-20</td>\n",
       "      <td>Non cancerous tissue</td>\n",
       "      <td>Non cancerous tissue</td>\n",
       "      <td>CELLC</td>\n",
       "      <td>Control Analyte</td>\n",
       "      <td>NAAEMA</td>\n",
       "      <td>TARGET</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TARGET-00-NAAEMB-20</td>\n",
       "      <td>Non cancerous tissue</td>\n",
       "      <td>Non cancerous tissue</td>\n",
       "      <td>CELLC</td>\n",
       "      <td>Control Analyte</td>\n",
       "      <td>NAAEMB</td>\n",
       "      <td>TARGET</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TARGET-00-NAAEMC-20</td>\n",
       "      <td>Non cancerous tissue</td>\n",
       "      <td>Non cancerous tissue</td>\n",
       "      <td>CELLC</td>\n",
       "      <td>Control Analyte</td>\n",
       "      <td>NAAEMC</td>\n",
       "      <td>TARGET</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             sample_id  primary_disease_code      _primary_disease  \\\n",
       "0  TARGET-00-NAAEMA-20  Non cancerous tissue  Non cancerous tissue   \n",
       "1  TARGET-00-NAAEMB-20  Non cancerous tissue  Non cancerous tissue   \n",
       "2  TARGET-00-NAAEMC-20  Non cancerous tissue  Non cancerous tissue   \n",
       "\n",
       "  sample_type_code     _sample_type _PATIENT _cohort  \n",
       "0            CELLC  Control Analyte   NAAEMA  TARGET  \n",
       "1            CELLC  Control Analyte   NAAEMB  TARGET  \n",
       "2            CELLC  Control Analyte   NAAEMC  TARGET  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = os.path.join('download', 'TARGET_phenotype.gz')\n",
    "pheno_df = pd.read_table(path)\n",
    "\n",
    "print(pheno_df.shape)\n",
    "pheno_df.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read Entrez ID Curation Information\n",
    "\n",
    "Load curated gene names from versioned resource. See https://github.com/cognoma/genes for more details"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Commit from https://github.com/cognoma/genes\n",
    "genes_commit = 'ad9631bb4e77e2cdc5413b0d77cb8f7e93fc5bee'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20395, 7)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>entrez_gene_id</th>\n",
       "      <th>symbol</th>\n",
       "      <th>description</th>\n",
       "      <th>chromosome</th>\n",
       "      <th>gene_type</th>\n",
       "      <th>synonyms</th>\n",
       "      <th>aliases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>A1BG</td>\n",
       "      <td>alpha-1-B glycoprotein</td>\n",
       "      <td>19</td>\n",
       "      <td>protein-coding</td>\n",
       "      <td>A1B|ABG|GAB|HYST2477</td>\n",
       "      <td>alpha-1B-glycoprotein|HEL-S-163pA|epididymis s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>A2M</td>\n",
       "      <td>alpha-2-macroglobulin</td>\n",
       "      <td>12</td>\n",
       "      <td>protein-coding</td>\n",
       "      <td>A2MD|CPAMD5|FWP007|S863-7</td>\n",
       "      <td>alpha-2-macroglobulin|C3 and PZP-like alpha-2-...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   entrez_gene_id symbol             description chromosome       gene_type  \\\n",
       "0               1   A1BG  alpha-1-B glycoprotein         19  protein-coding   \n",
       "1               2    A2M   alpha-2-macroglobulin         12  protein-coding   \n",
       "\n",
       "                    synonyms  \\\n",
       "0       A1B|ABG|GAB|HYST2477   \n",
       "1  A2MD|CPAMD5|FWP007|S863-7   \n",
       "\n",
       "                                             aliases  \n",
       "0  alpha-1B-glycoprotein|HEL-S-163pA|epididymis s...  \n",
       "1  alpha-2-macroglobulin|C3 and PZP-like alpha-2-...  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = 'https://raw.githubusercontent.com/cognoma/genes/{}/data/genes.tsv'.format(genes_commit)\n",
    "gene_df = pd.read_table(url)\n",
    "\n",
    "# Only consider protein-coding genes\n",
    "gene_df = (\n",
    "    gene_df.query(\"gene_type == 'protein-coding'\")\n",
    ")\n",
    "\n",
    "print(gene_df.shape)\n",
    "gene_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load gene updater - old to new Entrez gene identifiers\n",
    "url = 'https://raw.githubusercontent.com/cognoma/genes/{}/data/updater.tsv'.format(genes_commit)\n",
    "updater_df = pd.read_table(url)\n",
    "old_to_new_entrez = dict(zip(updater_df.old_entrez_gene_id,\n",
    "                             updater_df.new_entrez_gene_id))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read Probe Mapping Info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(18855, 13)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>gene</th>\n",
       "      <th>chrom</th>\n",
       "      <th>chromStart</th>\n",
       "      <th>chromEnd</th>\n",
       "      <th>strand</th>\n",
       "      <th>entrez_gene_id</th>\n",
       "      <th>symbol</th>\n",
       "      <th>description</th>\n",
       "      <th>chromosome</th>\n",
       "      <th>gene_type</th>\n",
       "      <th>synonyms</th>\n",
       "      <th>aliases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ENSG00000186092.4</td>\n",
       "      <td>OR4F5</td>\n",
       "      <td>chr1</td>\n",
       "      <td>69091</td>\n",
       "      <td>70008</td>\n",
       "      <td>+</td>\n",
       "      <td>79501</td>\n",
       "      <td>OR4F5</td>\n",
       "      <td>olfactory receptor family 4 subfamily F member 5</td>\n",
       "      <td>1</td>\n",
       "      <td>protein-coding</td>\n",
       "      <td>NaN</td>\n",
       "      <td>olfactory receptor 4F5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ENSG00000278566.1</td>\n",
       "      <td>OR4F29</td>\n",
       "      <td>chr1</td>\n",
       "      <td>450740</td>\n",
       "      <td>451678</td>\n",
       "      <td>-</td>\n",
       "      <td>729759</td>\n",
       "      <td>OR4F29</td>\n",
       "      <td>olfactory receptor family 4 subfamily F member 29</td>\n",
       "      <td>1</td>\n",
       "      <td>protein-coding</td>\n",
       "      <td>OR7-21</td>\n",
       "      <td>olfactory receptor 4F3/4F16/4F29|olfactory rec...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ENSG00000273547.1</td>\n",
       "      <td>OR4F16</td>\n",
       "      <td>chr1</td>\n",
       "      <td>685716</td>\n",
       "      <td>686654</td>\n",
       "      <td>-</td>\n",
       "      <td>81399</td>\n",
       "      <td>OR4F16</td>\n",
       "      <td>olfactory receptor family 4 subfamily F member 16</td>\n",
       "      <td>1</td>\n",
       "      <td>protein-coding</td>\n",
       "      <td>OR1-1|OR7-21</td>\n",
       "      <td>olfactory receptor 4F3/4F16/4F29|olfactory rec...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  id    gene chrom  chromStart  chromEnd strand  \\\n",
       "0  ENSG00000186092.4   OR4F5  chr1       69091     70008      +   \n",
       "1  ENSG00000278566.1  OR4F29  chr1      450740    451678      -   \n",
       "2  ENSG00000273547.1  OR4F16  chr1      685716    686654      -   \n",
       "\n",
       "   entrez_gene_id  symbol                                        description  \\\n",
       "0           79501   OR4F5   olfactory receptor family 4 subfamily F member 5   \n",
       "1          729759  OR4F29  olfactory receptor family 4 subfamily F member 29   \n",
       "2           81399  OR4F16  olfactory receptor family 4 subfamily F member 16   \n",
       "\n",
       "  chromosome       gene_type      synonyms  \\\n",
       "0          1  protein-coding           NaN   \n",
       "1          1  protein-coding        OR7-21   \n",
       "2          1  protein-coding  OR1-1|OR7-21   \n",
       "\n",
       "                                             aliases  \n",
       "0                             olfactory receptor 4F5  \n",
       "1  olfactory receptor 4F3/4F16/4F29|olfactory rec...  \n",
       "2  olfactory receptor 4F3/4F16/4F29|olfactory rec...  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = os.path.join('download', 'gencode.v23.annotation.gene.probemap')\n",
    "probe_map_df = pd.read_table(path)\n",
    "\n",
    "# Inner merge gene df to get ensembl to entrez mapping\n",
    "probe_map_df = probe_map_df.merge(gene_df, how='inner', left_on='gene', right_on='symbol')\n",
    "\n",
    "# Mapping to rename gene expression index\n",
    "ensembl_to_entrez = dict(zip(probe_map_df.id, probe_map_df.entrez_gene_id))\n",
    "\n",
    "print(probe_map_df.shape)\n",
    "probe_map_df.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read Gene Expression Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60498, 734)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TARGET-30-PASWYR-01</th>\n",
       "      <th>TARGET-20-PARUBT-09</th>\n",
       "      <th>TARGET-30-PASNZU-01</th>\n",
       "      <th>TARGET-52-PASDLA-11</th>\n",
       "      <th>TARGET-50-PAKNRX-01</th>\n",
       "      <th>TARGET-30-PASWFB-01</th>\n",
       "      <th>TARGET-30-PALUYS-01</th>\n",
       "      <th>TARGET-30-PAUDDK-01</th>\n",
       "      <th>TARGET-10-PAPZNK-09</th>\n",
       "      <th>TARGET-50-PAJLNJ-01</th>\n",
       "      <th>...</th>\n",
       "      <th>TARGET-20-PANLIZ-04</th>\n",
       "      <th>TARGET-21-PASSLT-41</th>\n",
       "      <th>TARGET-20-PASTTW-09</th>\n",
       "      <th>TARGET-50-PAKYLT-01</th>\n",
       "      <th>TARGET-20-PATJHJ-09</th>\n",
       "      <th>TARGET-21-PATKKJ-41</th>\n",
       "      <th>TARGET-10-PAPEJN-04</th>\n",
       "      <th>TARGET-20-PABLDZ-09</th>\n",
       "      <th>TARGET-10-PANSBR-09</th>\n",
       "      <th>TARGET-10-PARFLV-04</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sample</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ENSG00000242268.2</th>\n",
       "      <td>-3.0469</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-3.0469</td>\n",
       "      <td>-4.6082</td>\n",
       "      <td>-5.5735</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-2.2447</td>\n",
       "      <td>...</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-4.0350</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-4.2934</td>\n",
       "      <td>-9.9658</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000259041.1</th>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>...</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-9.9658</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 734 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   TARGET-30-PASWYR-01  TARGET-20-PARUBT-09  \\\n",
       "sample                                                        \n",
       "ENSG00000242268.2              -3.0469              -9.9658   \n",
       "ENSG00000259041.1              -9.9658              -9.9658   \n",
       "\n",
       "                   TARGET-30-PASNZU-01  TARGET-52-PASDLA-11  \\\n",
       "sample                                                        \n",
       "ENSG00000242268.2              -3.0469              -4.6082   \n",
       "ENSG00000259041.1              -9.9658              -9.9658   \n",
       "\n",
       "                   TARGET-50-PAKNRX-01  TARGET-30-PASWFB-01  \\\n",
       "sample                                                        \n",
       "ENSG00000242268.2              -5.5735              -9.9658   \n",
       "ENSG00000259041.1              -9.9658              -9.9658   \n",
       "\n",
       "                   TARGET-30-PALUYS-01  TARGET-30-PAUDDK-01  \\\n",
       "sample                                                        \n",
       "ENSG00000242268.2              -9.9658              -9.9658   \n",
       "ENSG00000259041.1              -9.9658              -9.9658   \n",
       "\n",
       "                   TARGET-10-PAPZNK-09  TARGET-50-PAJLNJ-01  \\\n",
       "sample                                                        \n",
       "ENSG00000242268.2              -9.9658              -2.2447   \n",
       "ENSG00000259041.1              -9.9658              -9.9658   \n",
       "\n",
       "                          ...           TARGET-20-PANLIZ-04  \\\n",
       "sample                    ...                                 \n",
       "ENSG00000242268.2         ...                       -9.9658   \n",
       "ENSG00000259041.1         ...                       -9.9658   \n",
       "\n",
       "                   TARGET-21-PASSLT-41  TARGET-20-PASTTW-09  \\\n",
       "sample                                                        \n",
       "ENSG00000242268.2              -9.9658              -4.0350   \n",
       "ENSG00000259041.1              -9.9658              -9.9658   \n",
       "\n",
       "                   TARGET-50-PAKYLT-01  TARGET-20-PATJHJ-09  \\\n",
       "sample                                                        \n",
       "ENSG00000242268.2              -9.9658              -9.9658   \n",
       "ENSG00000259041.1              -9.9658              -9.9658   \n",
       "\n",
       "                   TARGET-21-PATKKJ-41  TARGET-10-PAPEJN-04  \\\n",
       "sample                                                        \n",
       "ENSG00000242268.2              -9.9658              -9.9658   \n",
       "ENSG00000259041.1              -9.9658              -9.9658   \n",
       "\n",
       "                   TARGET-20-PABLDZ-09  TARGET-10-PANSBR-09  \\\n",
       "sample                                                        \n",
       "ENSG00000242268.2              -9.9658              -4.2934   \n",
       "ENSG00000259041.1              -9.9658              -9.9658   \n",
       "\n",
       "                   TARGET-10-PARFLV-04  \n",
       "sample                                  \n",
       "ENSG00000242268.2              -9.9658  \n",
       "ENSG00000259041.1              -9.9658  \n",
       "\n",
       "[2 rows x 734 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file = os.path.join('download', 'target_RSEM_gene_fpkm.gz')\n",
    "expr_df = pd.read_table(file, index_col=0)\n",
    "\n",
    "print(expr_df.shape)\n",
    "expr_df.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Process gene expression matrix\n",
    "\n",
    "This involves updating Entrez gene ids, sorting and subsetting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(734, 18753)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>id</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>18</th>\n",
       "      <th>...</th>\n",
       "      <th>102724231</th>\n",
       "      <th>102724398</th>\n",
       "      <th>102724473</th>\n",
       "      <th>102724536</th>\n",
       "      <th>102724631</th>\n",
       "      <th>102724862</th>\n",
       "      <th>102724928</th>\n",
       "      <th>105375355</th>\n",
       "      <th>105378803</th>\n",
       "      <th>105378952</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sample_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>TARGET-10-PAKSWW-03</th>\n",
       "      <td>5.3754</td>\n",
       "      <td>-1.1488</td>\n",
       "      <td>-1.4305</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-7.76965</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>4.3786</td>\n",
       "      <td>-1.3183</td>\n",
       "      <td>2.0289</td>\n",
       "      <td>0.7321</td>\n",
       "      <td>...</td>\n",
       "      <td>-2.6349</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-4.6082</td>\n",
       "      <td>0.1648</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-5.0116</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-0.8863</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-9.9658</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TARGET-10-PAMXHJ-09</th>\n",
       "      <td>4.9388</td>\n",
       "      <td>-1.2828</td>\n",
       "      <td>0.2881</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-9.96580</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>3.8660</td>\n",
       "      <td>1.4011</td>\n",
       "      <td>3.0791</td>\n",
       "      <td>2.6232</td>\n",
       "      <td>...</td>\n",
       "      <td>-2.2447</td>\n",
       "      <td>-4.2934</td>\n",
       "      <td>-4.6082</td>\n",
       "      <td>0.4016</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-4.2934</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-2.0529</td>\n",
       "      <td>-9.9658</td>\n",
       "      <td>-9.9658</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 18753 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "id                   1          2          9          10         12         \\\n",
       "sample_id                                                                    \n",
       "TARGET-10-PAKSWW-03     5.3754    -1.1488    -1.4305    -9.9658   -7.76965   \n",
       "TARGET-10-PAMXHJ-09     4.9388    -1.2828     0.2881    -9.9658   -9.96580   \n",
       "\n",
       "id                   13         14         15         16         18         \\\n",
       "sample_id                                                                    \n",
       "TARGET-10-PAKSWW-03    -9.9658     4.3786    -1.3183     2.0289     0.7321   \n",
       "TARGET-10-PAMXHJ-09    -9.9658     3.8660     1.4011     3.0791     2.6232   \n",
       "\n",
       "id                     ...      102724231  102724398  102724473  102724536  \\\n",
       "sample_id              ...                                                   \n",
       "TARGET-10-PAKSWW-03    ...        -2.6349    -9.9658    -4.6082     0.1648   \n",
       "TARGET-10-PAMXHJ-09    ...        -2.2447    -4.2934    -4.6082     0.4016   \n",
       "\n",
       "id                   102724631  102724862  102724928  105375355  105378803  \\\n",
       "sample_id                                                                    \n",
       "TARGET-10-PAKSWW-03    -9.9658    -5.0116    -9.9658    -0.8863    -9.9658   \n",
       "TARGET-10-PAMXHJ-09    -9.9658    -4.2934    -9.9658    -2.0529    -9.9658   \n",
       "\n",
       "id                   105378952  \n",
       "sample_id                       \n",
       "TARGET-10-PAKSWW-03    -9.9658  \n",
       "TARGET-10-PAMXHJ-09    -9.9658  \n",
       "\n",
       "[2 rows x 18753 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "expr_df = (expr_df\n",
    "    .dropna(axis='rows')\n",
    "    .reindex(probe_map_df.id)\n",
    "    .rename(index=ensembl_to_entrez)\n",
    "    .rename(index=old_to_new_entrez)\n",
    "    .groupby(level=0).mean()\n",
    "    .transpose()\n",
    "    .sort_index(axis='rows')\n",
    "    .sort_index(axis='columns')\n",
    ")\n",
    "\n",
    "expr_df.index.rename('sample_id', inplace=True)\n",
    "\n",
    "print(expr_df.shape)\n",
    "expr_df.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stratify Balanced Training and Testing Sets in TARGET Gene Expression\n",
    "\n",
    "Output training and testing gene expression datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "strat = pheno_df.set_index('sample_id').reindex(expr_df.index).primary_disease_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cancertype</th>\n",
       "      <th>n =</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AML</td>\n",
       "      <td>196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ALL</td>\n",
       "      <td>194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NBL</td>\n",
       "      <td>162</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>WT</td>\n",
       "      <td>132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AML-IF</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>CCSK</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>RT</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  cancertype  n =\n",
       "0        AML  196\n",
       "1        ALL  194\n",
       "2        NBL  162\n",
       "3         WT  132\n",
       "4     AML-IF   32\n",
       "5       CCSK   13\n",
       "6         RT    5"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cancertype_count_df = (\n",
    "    pd.DataFrame(strat.value_counts())\n",
    "    .reset_index()\n",
    "    .rename({'index': 'cancertype', 'primary_disease_code': 'n ='}, axis='columns')\n",
    ")\n",
    "\n",
    "file = os.path.join('data', 'target_sample_counts.tsv')\n",
    "cancertype_count_df.to_csv(file, sep='\\t', index=False)\n",
    "\n",
    "cancertype_count_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df, test_df = train_test_split(expr_df,\n",
    "                                     test_size=0.1,\n",
    "                                     random_state=123,\n",
    "                                     stratify=strat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(660, 18753)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(74, 18753)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(train_df.shape)\n",
    "test_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_file = os.path.join('data', 'train_target_expression_matrix_processed.tsv.gz')\n",
    "train_df.to_csv(train_file, sep='\\t', compression='gzip', float_format='%.3g')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_file = os.path.join('data', 'test_target_expression_matrix_processed.tsv.gz')\n",
    "test_df.to_csv(test_file, sep='\\t', compression='gzip', float_format='%.3g')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sort genes based on median absolute deviation and output to file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Determine most variably expressed genes and subset\n",
    "mad_genes_df = pd.DataFrame(train_df.mad(axis=0).sort_values(ascending=False)).reset_index()\n",
    "mad_genes_df.columns = ['gene_id', 'median_absolute_deviation']\n",
    "\n",
    "file = os.path.join('data', 'target_mad_genes.tsv')\n",
    "mad_genes_df.to_csv(file, sep='\\t', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:biobombe]",
   "language": "python",
   "name": "conda-env-biobombe-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}