pyspark-test.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RDD Example:\n",
    "\n",
    "Read the simple list of [1,2,3,4,5] and compute summation  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark import SparkContext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc = SparkContext.getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = [1, 2, 3, 4, 5]\n",
    "distData = sc.parallelize(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 2, 3, 4, 5]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "distData.collect()# fetch the entire rdd to a single machine \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 2]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "distData.take(2) # fetch only the first two elements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "distData.reduce(lambda a, b: a + b)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DataFrame Spark SQL \n",
    "\n",
    "Read a CSV data from Kaggle's <a href=\"https://www.kaggle.com/mlg-ulb/creditcardfraud/data\"> Credit Card Fraud dataset </a> in to a spark DataFrame. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyspark as spark\n",
    "from pyspark.sql import SparkSession\n",
    "spark = SparkSession \\\n",
    "    .builder \\\n",
    "    .appName(\"Python Spark SQL basic example\") \\\n",
    "    .config(\"spark.some.config.option\", \"some-value\") \\\n",
    "    .getOrCreate()\n",
    "#For pandas like visualization - no need to use df.show()\n",
    "spark.conf.set(\"spark.sql.repl.eagerEval.enabled\", True) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = spark.read.csv('creditcard.csv', header=True)\n",
    "#header=True says that the first line is column name not values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border='1'>\n",
       "<tr><th>Time</th><th>V1</th><th>V2</th><th>V3</th><th>V4</th><th>V5</th><th>V6</th><th>V7</th><th>V8</th><th>V9</th><th>V10</th><th>V11</th><th>V12</th><th>V13</th><th>V14</th><th>V15</th><th>V16</th><th>V17</th><th>V18</th><th>V19</th><th>V20</th><th>V21</th><th>V22</th><th>V23</th><th>V24</th><th>V25</th><th>V26</th><th>V27</th><th>V28</th><th>Amount</th><th>Class</th></tr>\n",
       "<tr><td>0</td><td>-1.3598071336738</td><td>-0.0727811733098497</td><td>2.53634673796914</td><td>1.37815522427443</td><td>-0.338320769942518</td><td>0.462387777762292</td><td>0.239598554061257</td><td>0.0986979012610507</td><td>0.363786969611213</td><td>0.0907941719789316</td><td>-0.551599533260813</td><td>-0.617800855762348</td><td>-0.991389847235408</td><td>-0.311169353699879</td><td>1.46817697209427</td><td>-0.470400525259478</td><td>0.207971241929242</td><td>0.0257905801985591</td><td>0.403992960255733</td><td>0.251412098239705</td><td>-0.018306777944153</td><td>0.277837575558899</td><td>-0.110473910188767</td><td>0.0669280749146731</td><td>0.128539358273528</td><td>-0.189114843888824</td><td>0.133558376740387</td><td>-0.0210530534538215</td><td>149.62</td><td>0</td></tr>\n",
       "<tr><td>0</td><td>1.19185711131486</td><td>0.26615071205963</td><td>0.16648011335321</td><td>0.448154078460911</td><td>0.0600176492822243</td><td>-0.0823608088155687</td><td>-0.0788029833323113</td><td>0.0851016549148104</td><td>-0.255425128109186</td><td>-0.166974414004614</td><td>1.61272666105479</td><td>1.06523531137287</td><td>0.48909501589608</td><td>-0.143772296441519</td><td>0.635558093258208</td><td>0.463917041022171</td><td>-0.114804663102346</td><td>-0.183361270123994</td><td>-0.145783041325259</td><td>-0.0690831352230203</td><td>-0.225775248033138</td><td>-0.638671952771851</td><td>0.101288021253234</td><td>-0.339846475529127</td><td>0.167170404418143</td><td>0.125894532368176</td><td>-0.00898309914322813</td><td>0.0147241691924927</td><td>2.69</td><td>0</td></tr>\n",
       "<tr><td>1</td><td>-1.35835406159823</td><td>-1.34016307473609</td><td>1.77320934263119</td><td>0.379779593034328</td><td>-0.503198133318193</td><td>1.80049938079263</td><td>0.791460956450422</td><td>0.247675786588991</td><td>-1.51465432260583</td><td>0.207642865216696</td><td>0.624501459424895</td><td>0.066083685268831</td><td>0.717292731410831</td><td>-0.165945922763554</td><td>2.34586494901581</td><td>-2.89008319444231</td><td>1.10996937869599</td><td>-0.121359313195888</td><td>-2.26185709530414</td><td>0.524979725224404</td><td>0.247998153469754</td><td>0.771679401917229</td><td>0.909412262347719</td><td>-0.689280956490685</td><td>-0.327641833735251</td><td>-0.139096571514147</td><td>-0.0553527940384261</td><td>-0.0597518405929204</td><td>378.66</td><td>0</td></tr>\n",
       "<tr><td>1</td><td>-0.966271711572087</td><td>-0.185226008082898</td><td>1.79299333957872</td><td>-0.863291275036453</td><td>-0.0103088796030823</td><td>1.24720316752486</td><td>0.23760893977178</td><td>0.377435874652262</td><td>-1.38702406270197</td><td>-0.0549519224713749</td><td>-0.226487263835401</td><td>0.178228225877303</td><td>0.507756869957169</td><td>-0.28792374549456</td><td>-0.631418117709045</td><td>-1.0596472454325</td><td>-0.684092786345479</td><td>1.96577500349538</td><td>-1.2326219700892</td><td>-0.208037781160366</td><td>-0.108300452035545</td><td>0.00527359678253453</td><td>-0.190320518742841</td><td>-1.17557533186321</td><td>0.647376034602038</td><td>-0.221928844458407</td><td>0.0627228487293033</td><td>0.0614576285006353</td><td>123.5</td><td>0</td></tr>\n",
       "<tr><td>2</td><td>-1.15823309349523</td><td>0.877736754848451</td><td>1.548717846511</td><td>0.403033933955121</td><td>-0.407193377311653</td><td>0.0959214624684256</td><td>0.592940745385545</td><td>-0.270532677192282</td><td>0.817739308235294</td><td>0.753074431976354</td><td>-0.822842877946363</td><td>0.53819555014995</td><td>1.3458515932154</td><td>-1.11966983471731</td><td>0.175121130008994</td><td>-0.451449182813529</td><td>-0.237033239362776</td><td>-0.0381947870352842</td><td>0.803486924960175</td><td>0.408542360392758</td><td>-0.00943069713232919</td><td>0.79827849458971</td><td>-0.137458079619063</td><td>0.141266983824769</td><td>-0.206009587619756</td><td>0.502292224181569</td><td>0.219422229513348</td><td>0.215153147499206</td><td>69.99</td><td>0</td></tr>\n",
       "<tr><td>2</td><td>-0.425965884412454</td><td>0.960523044882985</td><td>1.14110934232219</td><td>-0.168252079760302</td><td>0.42098688077219</td><td>-0.0297275516639742</td><td>0.476200948720027</td><td>0.260314333074874</td><td>-0.56867137571251</td><td>-0.371407196834471</td><td>1.34126198001957</td><td>0.359893837038039</td><td>-0.358090652573631</td><td>-0.137133700217612</td><td>0.517616806555742</td><td>0.401725895589603</td><td>-0.0581328233640131</td><td>0.0686531494425432</td><td>-0.0331937877876282</td><td>0.0849676720682049</td><td>-0.208253514656728</td><td>-0.559824796253248</td><td>-0.0263976679795373</td><td>-0.371426583174346</td><td>-0.232793816737034</td><td>0.105914779097957</td><td>0.253844224739337</td><td>0.0810802569229443</td><td>3.67</td><td>0</td></tr>\n",
       "<tr><td>4</td><td>1.22965763450793</td><td>0.141003507049326</td><td>0.0453707735899449</td><td>1.20261273673594</td><td>0.191880988597645</td><td>0.272708122899098</td><td>-0.00515900288250983</td><td>0.0812129398830894</td><td>0.464959994783886</td><td>-0.0992543211289237</td><td>-1.41690724314928</td><td>-0.153825826253651</td><td>-0.75106271556262</td><td>0.16737196252175</td><td>0.0501435942254188</td><td>-0.443586797916727</td><td>0.00282051247234708</td><td>-0.61198733994012</td><td>-0.0455750446637976</td><td>-0.21963255278686</td><td>-0.167716265815783</td><td>-0.270709726172363</td><td>-0.154103786809305</td><td>-0.780055415004671</td><td>0.75013693580659</td><td>-0.257236845917139</td><td>0.0345074297438413</td><td>0.00516776890624916</td><td>4.99</td><td>0</td></tr>\n",
       "<tr><td>7</td><td>-0.644269442348146</td><td>1.41796354547385</td><td>1.0743803763556</td><td>-0.492199018495015</td><td>0.948934094764157</td><td>0.428118462833089</td><td>1.12063135838353</td><td>-3.80786423873589</td><td>0.615374730667027</td><td>1.24937617815176</td><td>-0.619467796121913</td><td>0.291474353088705</td><td>1.75796421396042</td><td>-1.32386521970526</td><td>0.686132504394383</td><td>-0.0761269994382006</td><td>-1.2221273453247</td><td>-0.358221569869078</td><td>0.324504731321494</td><td>-0.156741852488285</td><td>1.94346533978412</td><td>-1.01545470979971</td><td>0.057503529867291</td><td>-0.649709005559993</td><td>-0.415266566234811</td><td>-0.0516342969262494</td><td>-1.20692108094258</td><td>-1.08533918832377</td><td>40.8</td><td>0</td></tr>\n",
       "<tr><td>7</td><td>-0.89428608220282</td><td>0.286157196276544</td><td>-0.113192212729871</td><td>-0.271526130088604</td><td>2.6695986595986</td><td>3.72181806112751</td><td>0.370145127676916</td><td>0.851084443200905</td><td>-0.392047586798604</td><td>-0.410430432848439</td><td>-0.705116586646536</td><td>-0.110452261733098</td><td>-0.286253632470583</td><td>0.0743553603016731</td><td>-0.328783050303565</td><td>-0.210077268148783</td><td>-0.499767968800267</td><td>0.118764861004217</td><td>0.57032816746536</td><td>0.0527356691149697</td><td>-0.0734251001059225</td><td>-0.268091632235551</td><td>-0.204232669947878</td><td>1.0115918018785</td><td>0.373204680146282</td><td>-0.384157307702294</td><td>0.0117473564581996</td><td>0.14240432992147</td><td>93.2</td><td>0</td></tr>\n",
       "<tr><td>9</td><td>-0.33826175242575</td><td>1.11959337641566</td><td>1.04436655157316</td><td>-0.222187276738296</td><td>0.49936080649727</td><td>-0.24676110061991</td><td>0.651583206489972</td><td>0.0695385865186387</td><td>-0.736727316364109</td><td>-0.366845639206541</td><td>1.01761446783262</td><td>0.836389570307029</td><td>1.00684351373408</td><td>-0.443522816876142</td><td>0.150219101422635</td><td>0.739452777052119</td><td>-0.540979921943059</td><td>0.47667726004282</td><td>0.451772964394125</td><td>0.203711454727929</td><td>-0.246913936910008</td><td>-0.633752642406113</td><td>-0.12079408408185</td><td>-0.385049925313426</td><td>-0.0697330460416923</td><td>0.0941988339514961</td><td>0.246219304619926</td><td>0.0830756493473326</td><td>3.68</td><td>0</td></tr>\n",
       "<tr><td>10</td><td>1.44904378114715</td><td>-1.17633882535966</td><td>0.913859832832795</td><td>-1.37566665499943</td><td>-1.97138316545323</td><td>-0.62915213889734</td><td>-1.4232356010359</td><td>0.0484558879088564</td><td>-1.72040839292037</td><td>1.62665905834133</td><td>1.1996439495421</td><td>-0.671439778462005</td><td>-0.513947152539479</td><td>-0.0950450453999549</td><td>0.230930409124119</td><td>0.0319674667862076</td><td>0.253414715863197</td><td>0.854343814324194</td><td>-0.221365413645481</td><td>-0.387226474431156</td><td>-0.00930189652490052</td><td>0.313894410791098</td><td>0.0277401580170247</td><td>0.500512287104917</td><td>0.25136735874921</td><td>-0.129477953726618</td><td>0.0428498709381461</td><td>0.0162532619375515</td><td>7.8</td><td>0</td></tr>\n",
       "<tr><td>10</td><td>0.38497821518095</td><td>0.616109459176472</td><td>-0.874299702595052</td><td>-0.0940186259679115</td><td>2.92458437838817</td><td>3.31702716826156</td><td>0.470454671805879</td><td>0.53824722837695</td><td>-0.558894612428441</td><td>0.30975539423728</td><td>-0.259115563735702</td><td>-0.326143233995877</td><td>-0.0900467227020648</td><td>0.362832368569793</td><td>0.928903660629178</td><td>-0.129486811402759</td><td>-0.809978925963589</td><td>0.359985390219981</td><td>0.70766382644648</td><td>0.12599157561542</td><td>0.049923685888971</td><td>0.238421512225103</td><td>0.00912986861262866</td><td>0.996710209581086</td><td>-0.767314827174801</td><td>-0.492208295340017</td><td>0.042472441919027</td><td>-0.0543373883732122</td><td>9.99</td><td>0</td></tr>\n",
       "<tr><td>10</td><td>1.249998742053</td><td>-1.22163680921816</td><td>0.383930151282291</td><td>-1.23489868766892</td><td>-1.48541947377961</td><td>-0.753230164566149</td><td>-0.689404975426345</td><td>-0.227487227519552</td><td>-2.09401057344842</td><td>1.32372927445937</td><td>0.227666231237246</td><td>-0.242681998944186</td><td>1.20541680770748</td><td>-0.317630527025074</td><td>0.725674990179153</td><td>-0.815612186027305</td><td>0.873936447614439</td><td>-0.847788598847099</td><td>-0.683192626267037</td><td>-0.102755941505071</td><td>-0.231809239223849</td><td>-0.483285330117712</td><td>0.0846676908596583</td><td>0.392830885335013</td><td>0.161134553588505</td><td>-0.354990039673962</td><td>0.0264155490776107</td><td>0.0424220887282304</td><td>121.5</td><td>0</td></tr>\n",
       "<tr><td>11</td><td>1.0693735878819</td><td>0.287722129331455</td><td>0.828612726634281</td><td>2.71252042961718</td><td>-0.178398016248009</td><td>0.337543730282968</td><td>-0.0967168617395962</td><td>0.115981735546597</td><td>-0.221082566236194</td><td>0.460230444301678</td><td>-0.773656930526689</td><td>0.32338724546722</td><td>-0.0110758870883779</td><td>-0.178485175177916</td><td>-0.65556427824926</td><td>-0.19992517131173</td><td>0.1240054151819</td><td>-0.980496201537345</td><td>-0.982916082135047</td><td>-0.153197231044512</td><td>-0.0368755317335273</td><td>0.0744124028162195</td><td>-0.0714074332998586</td><td>0.104743752596029</td><td>0.548264725394119</td><td>0.104094153162781</td><td>0.0214910583643189</td><td>0.021293311477486</td><td>27.5</td><td>0</td></tr>\n",
       "<tr><td>12</td><td>-2.7918547659339</td><td>-0.327770756658658</td><td>1.64175016056605</td><td>1.76747274389883</td><td>-0.136588446465306</td><td>0.80759646826532</td><td>-0.422911389711497</td><td>-1.90710747624096</td><td>0.755712908314791</td><td>1.1510869876677</td><td>0.844555470974377</td><td>0.7929439518176</td><td>0.370448092803246</td><td>-0.734975105820311</td><td>0.406795710431001</td><td>-0.303057623825763</td><td>-0.155868714793874</td><td>0.778265457041536</td><td>2.22186801373788</td><td>-1.58212204356551</td><td>1.15166304848789</td><td>0.222181966098225</td><td>1.02058620426601</td><td>0.0283166513238872</td><td>-0.232746324289105</td><td>-0.23555721754117</td><td>-0.16477751177654</td><td>-0.0301536365592253</td><td>58.8</td><td>0</td></tr>\n",
       "<tr><td>12</td><td>-0.752417042956605</td><td>0.345485415344747</td><td>2.05732291276727</td><td>-1.46864329840046</td><td>-1.1583936804082</td><td>-0.0778498291166733</td><td>-0.608581418236123</td><td>0.00360348436201849</td><td>-0.436166983515744</td><td>0.747730827192802</td><td>-0.793980602837221</td><td>-0.770406728847129</td><td>1.04762699748088</td><td>-1.06660368148653</td><td>1.10695345662141</td><td>1.66011355713381</td><td>-0.279265373246772</td><td>-0.419994141181313</td><td>0.432535348618175</td><td>0.263450864446125</td><td>0.499624954671111</td><td>1.35365048557231</td><td>-0.256573280448308</td><td>-0.0650837078816517</td><td>-0.0391243535426488</td><td>-0.0870864732146962</td><td>-0.180997500092721</td><td>0.129394059390202</td><td>15.99</td><td>0</td></tr>\n",
       "<tr><td>12</td><td>1.10321543528383</td><td>-0.0402962145973447</td><td>1.2673320885949</td><td>1.28909146962552</td><td>-0.735997163604068</td><td>0.288069162976262</td><td>-0.586056786337461</td><td>0.189379713679593</td><td>0.782332891785191</td><td>-0.267975066537173</td><td>-0.450311279515466</td><td>0.936707714991982</td><td>0.708380406186981</td><td>-0.468647287707221</td><td>0.354574063407955</td><td>-0.246634655717582</td><td>-0.00921237772707382</td><td>-0.595912405700819</td><td>-0.57568162226261</td><td>-0.113910176982092</td><td>-0.0246120063374677</td><td>0.196001952806192</td><td>0.0138016541409422</td><td>0.103758331023198</td><td>0.364297540595235</td><td>-0.382260574113217</td><td>0.092809187460487</td><td>0.0370505169810008</td><td>12.99</td><td>0</td></tr>\n",
       "<tr><td>13</td><td>-0.436905071360625</td><td>0.918966212909322</td><td>0.92459077438817</td><td>-0.727219053596792</td><td>0.915678718106307</td><td>-0.127867352079254</td><td>0.707641607333935</td><td>0.0879623554672504</td><td>-0.66527135413364</td><td>-0.737979823596458</td><td>0.32409781346169</td><td>0.277192107214981</td><td>0.252624256310781</td><td>-0.291896460370468</td><td>-0.184520169327133</td><td>1.14317370716197</td><td>-0.92870926272403</td><td>0.680469592634687</td><td>0.0254364616880793</td><td>-0.0470212823165035</td><td>-0.194795823794671</td><td>-0.672637997017793</td><td>-0.156857514491897</td><td>-0.888386320943716</td><td>-0.342413218776576</td><td>-0.049026728633951</td><td>0.0796923991551505</td><td>0.131023789452311</td><td>0.89</td><td>0</td></tr>\n",
       "<tr><td>14</td><td>-5.40125766315825</td><td>-5.45014783420644</td><td>1.18630463143652</td><td>1.73623880012095</td><td>3.04910587764025</td><td>-1.76340557365201</td><td>-1.55973769907953</td><td>0.160841747266769</td><td>1.23308974041888</td><td>0.345172827050629</td><td>0.917229867699146</td><td>0.970116716069048</td><td>-0.266567764915222</td><td>-0.479129929276704</td><td>-0.526608502569153</td><td>0.47200411177674</td><td>-0.725480944982201</td><td>0.075081351540202</td><td>-0.406866573198217</td><td>-2.19684802485647</td><td>-0.503600328973703</td><td>0.984459785590244</td><td>2.45858857639219</td><td>0.0421188969891572</td><td>-0.481630823956716</td><td>-0.621272013713977</td><td>0.392053289557744</td><td>0.949594245504846</td><td>46.8</td><td>0</td></tr>\n",
       "<tr><td>15</td><td>1.4929359769862</td><td>-1.02934573189487</td><td>0.45479473374366</td><td>-1.43802587991702</td><td>-1.55543410136344</td><td>-0.720961147043557</td><td>-1.08066413038614</td><td>-0.0531271179483221</td><td>-1.9786815953872</td><td>1.63807603690446</td><td>1.07754241162743</td><td>-0.63204651464934</td><td>-0.41695716661602</td><td>0.0520105153724404</td><td>-0.0429789228232019</td><td>-0.166432496451972</td><td>0.304241418614353</td><td>0.554432499062278</td><td>0.0542295152184719</td><td>-0.387910172646258</td><td>-0.177649846438814</td><td>-0.175073809074822</td><td>0.0400022190621329</td><td>0.295813862676508</td><td>0.33293059939425</td><td>-0.220384850672322</td><td>0.0222984359135846</td><td>0.00760225559997897</td><td>5</td><td>0</td></tr>\n",
       "</table>\n",
       "only showing top 20 rows\n"
      ],
      "text/plain": [
       "+----+------------------+-------------------+------------------+-------------------+-------------------+-------------------+--------------------+-------------------+------------------+-------------------+------------------+------------------+-------------------+-------------------+-------------------+-------------------+--------------------+-------------------+-------------------+-------------------+--------------------+-------------------+-------------------+-------------------+-------------------+-------------------+--------------------+-------------------+------+-----+\n",
       "|Time|                V1|                 V2|                V3|                 V4|                 V5|                 V6|                  V7|                 V8|                V9|                V10|               V11|               V12|                V13|                V14|                V15|                V16|                 V17|                V18|                V19|                V20|                 V21|                V22|                V23|                V24|                V25|                V26|                 V27|                V28|Amount|Class|\n",
       "+----+------------------+-------------------+------------------+-------------------+-------------------+-------------------+--------------------+-------------------+------------------+-------------------+------------------+------------------+-------------------+-------------------+-------------------+-------------------+--------------------+-------------------+-------------------+-------------------+--------------------+-------------------+-------------------+-------------------+-------------------+-------------------+--------------------+-------------------+------+-----+\n",
       "|   0|  -1.3598071336738|-0.0727811733098497|  2.53634673796914|   1.37815522427443| -0.338320769942518|  0.462387777762292|   0.239598554061257| 0.0986979012610507| 0.363786969611213| 0.0907941719789316|-0.551599533260813|-0.617800855762348| -0.991389847235408| -0.311169353699879|   1.46817697209427| -0.470400525259478|   0.207971241929242| 0.0257905801985591|  0.403992960255733|  0.251412098239705|  -0.018306777944153|  0.277837575558899| -0.110473910188767| 0.0669280749146731|  0.128539358273528| -0.189114843888824|   0.133558376740387|-0.0210530534538215|149.62|    0|\n",
       "|   0|  1.19185711131486|   0.26615071205963|  0.16648011335321|  0.448154078460911| 0.0600176492822243|-0.0823608088155687| -0.0788029833323113| 0.0851016549148104|-0.255425128109186| -0.166974414004614|  1.61272666105479|  1.06523531137287|   0.48909501589608| -0.143772296441519|  0.635558093258208|  0.463917041022171|  -0.114804663102346| -0.183361270123994| -0.145783041325259|-0.0690831352230203|  -0.225775248033138| -0.638671952771851|  0.101288021253234| -0.339846475529127|  0.167170404418143|  0.125894532368176|-0.00898309914322813| 0.0147241691924927|  2.69|    0|\n",
       "|   1| -1.35835406159823|  -1.34016307473609|  1.77320934263119|  0.379779593034328| -0.503198133318193|   1.80049938079263|   0.791460956450422|  0.247675786588991| -1.51465432260583|  0.207642865216696| 0.624501459424895| 0.066083685268831|  0.717292731410831| -0.165945922763554|   2.34586494901581|  -2.89008319444231|    1.10996937869599| -0.121359313195888|  -2.26185709530414|  0.524979725224404|   0.247998153469754|  0.771679401917229|  0.909412262347719| -0.689280956490685| -0.327641833735251| -0.139096571514147| -0.0553527940384261|-0.0597518405929204|378.66|    0|\n",
       "|   1|-0.966271711572087| -0.185226008082898|  1.79299333957872| -0.863291275036453|-0.0103088796030823|   1.24720316752486|    0.23760893977178|  0.377435874652262| -1.38702406270197|-0.0549519224713749|-0.226487263835401| 0.178228225877303|  0.507756869957169|  -0.28792374549456| -0.631418117709045|   -1.0596472454325|  -0.684092786345479|   1.96577500349538|   -1.2326219700892| -0.208037781160366|  -0.108300452035545|0.00527359678253453| -0.190320518742841|  -1.17557533186321|  0.647376034602038| -0.221928844458407|  0.0627228487293033| 0.0614576285006353| 123.5|    0|\n",
       "|   2| -1.15823309349523|  0.877736754848451|    1.548717846511|  0.403033933955121| -0.407193377311653| 0.0959214624684256|   0.592940745385545| -0.270532677192282| 0.817739308235294|  0.753074431976354|-0.822842877946363|  0.53819555014995|    1.3458515932154|  -1.11966983471731|  0.175121130008994| -0.451449182813529|  -0.237033239362776|-0.0381947870352842|  0.803486924960175|  0.408542360392758|-0.00943069713232919|   0.79827849458971| -0.137458079619063|  0.141266983824769| -0.206009587619756|  0.502292224181569|   0.219422229513348|  0.215153147499206| 69.99|    0|\n",
       "|   2|-0.425965884412454|  0.960523044882985|  1.14110934232219| -0.168252079760302|   0.42098688077219|-0.0297275516639742|   0.476200948720027|  0.260314333074874| -0.56867137571251| -0.371407196834471|  1.34126198001957| 0.359893837038039| -0.358090652573631| -0.137133700217612|  0.517616806555742|  0.401725895589603| -0.0581328233640131| 0.0686531494425432|-0.0331937877876282| 0.0849676720682049|  -0.208253514656728| -0.559824796253248|-0.0263976679795373| -0.371426583174346| -0.232793816737034|  0.105914779097957|   0.253844224739337| 0.0810802569229443|  3.67|    0|\n",
       "|   4|  1.22965763450793|  0.141003507049326|0.0453707735899449|   1.20261273673594|  0.191880988597645|  0.272708122899098|-0.00515900288250983| 0.0812129398830894| 0.464959994783886|-0.0992543211289237| -1.41690724314928|-0.153825826253651|  -0.75106271556262|   0.16737196252175| 0.0501435942254188| -0.443586797916727| 0.00282051247234708|  -0.61198733994012|-0.0455750446637976|  -0.21963255278686|  -0.167716265815783| -0.270709726172363| -0.154103786809305| -0.780055415004671|   0.75013693580659| -0.257236845917139|  0.0345074297438413|0.00516776890624916|  4.99|    0|\n",
       "|   7|-0.644269442348146|   1.41796354547385|   1.0743803763556| -0.492199018495015|  0.948934094764157|  0.428118462833089|    1.12063135838353|  -3.80786423873589| 0.615374730667027|   1.24937617815176|-0.619467796121913| 0.291474353088705|   1.75796421396042|  -1.32386521970526|  0.686132504394383|-0.0761269994382006|    -1.2221273453247| -0.358221569869078|  0.324504731321494| -0.156741852488285|    1.94346533978412|  -1.01545470979971|  0.057503529867291| -0.649709005559993| -0.415266566234811|-0.0516342969262494|   -1.20692108094258|  -1.08533918832377|  40.8|    0|\n",
       "|   7| -0.89428608220282|  0.286157196276544|-0.113192212729871| -0.271526130088604|    2.6695986595986|   3.72181806112751|   0.370145127676916|  0.851084443200905|-0.392047586798604| -0.410430432848439|-0.705116586646536|-0.110452261733098| -0.286253632470583| 0.0743553603016731| -0.328783050303565| -0.210077268148783|  -0.499767968800267|  0.118764861004217|   0.57032816746536| 0.0527356691149697| -0.0734251001059225| -0.268091632235551| -0.204232669947878|    1.0115918018785|  0.373204680146282| -0.384157307702294|  0.0117473564581996|   0.14240432992147|  93.2|    0|\n",
       "|   9| -0.33826175242575|   1.11959337641566|  1.04436655157316| -0.222187276738296|   0.49936080649727|  -0.24676110061991|   0.651583206489972| 0.0695385865186387|-0.736727316364109| -0.366845639206541|  1.01761446783262| 0.836389570307029|   1.00684351373408| -0.443522816876142|  0.150219101422635|  0.739452777052119|  -0.540979921943059|   0.47667726004282|  0.451772964394125|  0.203711454727929|  -0.246913936910008| -0.633752642406113|  -0.12079408408185| -0.385049925313426|-0.0697330460416923| 0.0941988339514961|   0.246219304619926| 0.0830756493473326|  3.68|    0|\n",
       "|  10|  1.44904378114715|  -1.17633882535966| 0.913859832832795|  -1.37566665499943|  -1.97138316545323|  -0.62915213889734|    -1.4232356010359| 0.0484558879088564| -1.72040839292037|   1.62665905834133|   1.1996439495421|-0.671439778462005| -0.513947152539479|-0.0950450453999549|  0.230930409124119| 0.0319674667862076|   0.253414715863197|  0.854343814324194| -0.221365413645481| -0.387226474431156|-0.00930189652490052|  0.313894410791098| 0.0277401580170247|  0.500512287104917|   0.25136735874921| -0.129477953726618|  0.0428498709381461| 0.0162532619375515|   7.8|    0|\n",
       "|  10|  0.38497821518095|  0.616109459176472|-0.874299702595052|-0.0940186259679115|   2.92458437838817|   3.31702716826156|   0.470454671805879|   0.53824722837695|-0.558894612428441|   0.30975539423728|-0.259115563735702|-0.326143233995877|-0.0900467227020648|  0.362832368569793|  0.928903660629178| -0.129486811402759|  -0.809978925963589|  0.359985390219981|   0.70766382644648|   0.12599157561542|   0.049923685888971|  0.238421512225103|0.00912986861262866|  0.996710209581086| -0.767314827174801| -0.492208295340017|   0.042472441919027|-0.0543373883732122|  9.99|    0|\n",
       "|  10|    1.249998742053|  -1.22163680921816| 0.383930151282291|  -1.23489868766892|  -1.48541947377961| -0.753230164566149|  -0.689404975426345| -0.227487227519552| -2.09401057344842|   1.32372927445937| 0.227666231237246|-0.242681998944186|   1.20541680770748| -0.317630527025074|  0.725674990179153| -0.815612186027305|   0.873936447614439| -0.847788598847099| -0.683192626267037| -0.102755941505071|  -0.231809239223849| -0.483285330117712| 0.0846676908596583|  0.392830885335013|  0.161134553588505| -0.354990039673962|  0.0264155490776107| 0.0424220887282304| 121.5|    0|\n",
       "|  11|   1.0693735878819|  0.287722129331455| 0.828612726634281|   2.71252042961718| -0.178398016248009|  0.337543730282968| -0.0967168617395962|  0.115981735546597|-0.221082566236194|  0.460230444301678|-0.773656930526689|  0.32338724546722|-0.0110758870883779| -0.178485175177916|  -0.65556427824926|  -0.19992517131173|     0.1240054151819| -0.980496201537345| -0.982916082135047| -0.153197231044512| -0.0368755317335273| 0.0744124028162195|-0.0714074332998586|  0.104743752596029|  0.548264725394119|  0.104094153162781|  0.0214910583643189|  0.021293311477486|  27.5|    0|\n",
       "|  12|  -2.7918547659339| -0.327770756658658|  1.64175016056605|   1.76747274389883| -0.136588446465306|   0.80759646826532|  -0.422911389711497|  -1.90710747624096| 0.755712908314791|    1.1510869876677| 0.844555470974377|   0.7929439518176|  0.370448092803246| -0.734975105820311|  0.406795710431001| -0.303057623825763|  -0.155868714793874|  0.778265457041536|   2.22186801373788|  -1.58212204356551|    1.15166304848789|  0.222181966098225|   1.02058620426601| 0.0283166513238872| -0.232746324289105|  -0.23555721754117|   -0.16477751177654|-0.0301536365592253|  58.8|    0|\n",
       "|  12|-0.752417042956605|  0.345485415344747|  2.05732291276727|  -1.46864329840046|   -1.1583936804082|-0.0778498291166733|  -0.608581418236123|0.00360348436201849|-0.436166983515744|  0.747730827192802|-0.793980602837221|-0.770406728847129|   1.04762699748088|  -1.06660368148653|   1.10695345662141|   1.66011355713381|  -0.279265373246772| -0.419994141181313|  0.432535348618175|  0.263450864446125|   0.499624954671111|   1.35365048557231| -0.256573280448308|-0.0650837078816517|-0.0391243535426488|-0.0870864732146962|  -0.180997500092721|  0.129394059390202| 15.99|    0|\n",
       "|  12|  1.10321543528383|-0.0402962145973447|   1.2673320885949|   1.28909146962552| -0.735997163604068|  0.288069162976262|  -0.586056786337461|  0.189379713679593| 0.782332891785191| -0.267975066537173|-0.450311279515466| 0.936707714991982|  0.708380406186981| -0.468647287707221|  0.354574063407955| -0.246634655717582|-0.00921237772707382| -0.595912405700819|  -0.57568162226261| -0.113910176982092| -0.0246120063374677|  0.196001952806192| 0.0138016541409422|  0.103758331023198|  0.364297540595235| -0.382260574113217|   0.092809187460487| 0.0370505169810008| 12.99|    0|\n",
       "|  13|-0.436905071360625|  0.918966212909322|  0.92459077438817| -0.727219053596792|  0.915678718106307| -0.127867352079254|   0.707641607333935| 0.0879623554672504| -0.66527135413364| -0.737979823596458|  0.32409781346169| 0.277192107214981|  0.252624256310781| -0.291896460370468| -0.184520169327133|   1.14317370716197|   -0.92870926272403|  0.680469592634687| 0.0254364616880793|-0.0470212823165035|  -0.194795823794671| -0.672637997017793| -0.156857514491897| -0.888386320943716| -0.342413218776576| -0.049026728633951|  0.0796923991551505|  0.131023789452311|  0.89|    0|\n",
       "|  14| -5.40125766315825|  -5.45014783420644|  1.18630463143652|   1.73623880012095|   3.04910587764025|  -1.76340557365201|   -1.55973769907953|  0.160841747266769|  1.23308974041888|  0.345172827050629| 0.917229867699146| 0.970116716069048| -0.266567764915222| -0.479129929276704| -0.526608502569153|   0.47200411177674|  -0.725480944982201|  0.075081351540202| -0.406866573198217|  -2.19684802485647|  -0.503600328973703|  0.984459785590244|   2.45858857639219| 0.0421188969891572| -0.481630823956716| -0.621272013713977|   0.392053289557744|  0.949594245504846|  46.8|    0|\n",
       "|  15|   1.4929359769862|  -1.02934573189487|  0.45479473374366|  -1.43802587991702|  -1.55543410136344| -0.720961147043557|   -1.08066413038614|-0.0531271179483221|  -1.9786815953872|   1.63807603690446|  1.07754241162743| -0.63204651464934|  -0.41695716661602| 0.0520105153724404|-0.0429789228232019| -0.166432496451972|   0.304241418614353|  0.554432499062278| 0.0542295152184719| -0.387910172646258|  -0.177649846438814| -0.175073809074822| 0.0400022190621329|  0.295813862676508|   0.33293059939425| -0.220384850672322|  0.0222984359135846|0.00760225559997897|     5|    0|\n",
       "+----+------------------+-------------------+------------------+-------------------+-------------------+-------------------+--------------------+-------------------+------------------+-------------------+------------------+------------------+-------------------+-------------------+-------------------+-------------------+--------------------+-------------------+-------------------+-------------------+--------------------+-------------------+-------------------+-------------------+-------------------+-------------------+--------------------+-------------------+------+-----+\n",
       "only showing top 20 rows"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- Time: string (nullable = true)\n",
      " |-- V1: string (nullable = true)\n",
      " |-- V2: string (nullable = true)\n",
      " |-- V3: string (nullable = true)\n",
      " |-- V4: string (nullable = true)\n",
      " |-- V5: string (nullable = true)\n",
      " |-- V6: string (nullable = true)\n",
      " |-- V7: string (nullable = true)\n",
      " |-- V8: string (nullable = true)\n",
      " |-- V9: string (nullable = true)\n",
      " |-- V10: string (nullable = true)\n",
      " |-- V11: string (nullable = true)\n",
      " |-- V12: string (nullable = true)\n",
      " |-- V13: string (nullable = true)\n",
      " |-- V14: string (nullable = true)\n",
      " |-- V15: string (nullable = true)\n",
      " |-- V16: string (nullable = true)\n",
      " |-- V17: string (nullable = true)\n",
      " |-- V18: string (nullable = true)\n",
      " |-- V19: string (nullable = true)\n",
      " |-- V20: string (nullable = true)\n",
      " |-- V21: string (nullable = true)\n",
      " |-- V22: string (nullable = true)\n",
      " |-- V23: string (nullable = true)\n",
      " |-- V24: string (nullable = true)\n",
      " |-- V25: string (nullable = true)\n",
      " |-- V26: string (nullable = true)\n",
      " |-- V27: string (nullable = true)\n",
      " |-- V28: string (nullable = true)\n",
      " |-- Amount: string (nullable = true)\n",
      " |-- Class: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.printSchema() # shows us the columns name and their data type.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(Time='0', V1='-1.3598071336738', V2='-0.0727811733098497', V3='2.53634673796914', V4='1.37815522427443', V5='-0.338320769942518', V6='0.462387777762292', V7='0.239598554061257', V8='0.0986979012610507', V9='0.363786969611213', V10='0.0907941719789316', V11='-0.551599533260813', V12='-0.617800855762348', V13='-0.991389847235408', V14='-0.311169353699879', V15='1.46817697209427', V16='-0.470400525259478', V17='0.207971241929242', V18='0.0257905801985591', V19='0.403992960255733', V20='0.251412098239705', V21='-0.018306777944153', V22='0.277837575558899', V23='-0.110473910188767', V24='0.0669280749146731', V25='0.128539358273528', V26='-0.189114843888824', V27='0.133558376740387', V28='-0.0210530534538215', Amount='149.62', Class='0'),\n",
       " Row(Time='0', V1='1.19185711131486', V2='0.26615071205963', V3='0.16648011335321', V4='0.448154078460911', V5='0.0600176492822243', V6='-0.0823608088155687', V7='-0.0788029833323113', V8='0.0851016549148104', V9='-0.255425128109186', V10='-0.166974414004614', V11='1.61272666105479', V12='1.06523531137287', V13='0.48909501589608', V14='-0.143772296441519', V15='0.635558093258208', V16='0.463917041022171', V17='-0.114804663102346', V18='-0.183361270123994', V19='-0.145783041325259', V20='-0.0690831352230203', V21='-0.225775248033138', V22='-0.638671952771851', V23='0.101288021253234', V24='-0.339846475529127', V25='0.167170404418143', V26='0.125894532368176', V27='-0.00898309914322813', V28='0.0147241691924927', Amount='2.69', Class='0')]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "284807"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.count()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Statistical summary on numerical columns can be computed by describe():**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border='1'>\n",
       "<tr><th>summary</th><th>Time</th><th>V1</th><th>V2</th><th>V3</th><th>V4</th><th>V5</th><th>V6</th><th>V7</th><th>V8</th><th>V9</th><th>V10</th><th>V11</th><th>V12</th><th>V13</th><th>V14</th><th>V15</th><th>V16</th><th>V17</th><th>V18</th><th>V19</th><th>V20</th><th>V21</th><th>V22</th><th>V23</th><th>V24</th><th>V25</th><th>V26</th><th>V27</th><th>V28</th><th>Amount</th><th>Class</th></tr>\n",
       "<tr><td>count</td><td>284807</td><td>284807</td><td>284807</td><td>284807</td><td>284807</td><td>284807</td><td>284807</td><td>284807</td><td>284807</td><td>284807</td><td>284807</td><td>284807</td><td>284807</td><td>284807</td><td>284807</td><td>284807</td><td>284807</td><td>284807</td><td>284807</td><td>284807</td><td>284807</td><td>284807</td><td>284807</td><td>284807</td><td>284807</td><td>284807</td><td>284807</td><td>284807</td><td>284807</td><td>284807</td><td>284807</td></tr>\n",
       "<tr><td>mean</td><td>94813.85957508067</td><td>2.235360406313924...</td><td>6.865749819392767...</td><td>-5.82471054445228...</td><td>2.011824365682531...</td><td>3.704311530463074...</td><td>1.140033807220101...</td><td>-1.14961392324716...</td><td>-2.9538691083434E-16</td><td>-2.08207854988096...</td><td>2.145945990061367...</td><td>-1.27734880360795...</td><td>-3.30833340134460...</td><td>1.034652530922445...</td><td>1.708454024825641...</td><td>5.933285192758958...</td><td>1.394705224939437...</td><td>-2.87403480811790...</td><td>1.749967860942900...</td><td>9.38053027649593E-16</td><td>3.81607955077877E-16</td><td>2.850084518050253...</td><td>-7.66409282164773...</td><td>2.9538691083434E-16</td><td>4.454953538333312...</td><td>1.034652530922445...</td><td>1.808246900107513...</td><td>-3.52368642620288...</td><td>-1.13364706320206...</td><td>88.3496192509521</td><td>0.001727485630620034</td></tr>\n",
       "<tr><td>stddev</td><td>47488.145954566324</td><td>1.9586958038574904</td><td>1.6513085794769997</td><td>1.5162550051777732</td><td>1.415868574940927</td><td>1.380246734031437</td><td>1.3322710897575714</td><td>1.2370935981826632</td><td>1.1943529026692048</td><td>1.0986320892243222</td><td>1.0888497654025215</td><td>1.0207130277115581</td><td>0.9992013895301415</td><td>0.9952742301251558</td><td>0.9585956112570617</td><td>0.9153160116104389</td><td>0.8762528873883704</td><td>0.84933706367439</td><td>0.8381762095288453</td><td>0.8140405007685797</td><td>0.7709250248871159</td><td>0.7345240143713125</td><td>0.7257015604409107</td><td>0.6244602955949898</td><td>0.6056470678271603</td><td>0.5212780705409427</td><td>0.48222701326105666</td><td>0.4036324949650313</td><td>0.33008326416025036</td><td>250.1201092401885</td><td>0.041527189635464985</td></tr>\n",
       "<tr><td>min</td><td>0</td><td>-0.00012931370800...</td><td>-0.00010296722561...</td><td>-0.00010859127517...</td><td>-0.00011921826106...</td><td>-0.00010366562678...</td><td>-0.00010234903761...</td><td>-0.00010533581684...</td><td>-0.00010065655617...</td><td>-0.00010181309940...</td><td>-0.00010546343208...</td><td>-0.00010815155961...</td><td>-0.00010708756424...</td><td>-0.0001012565441717</td><td>-0.00011026584350728</td><td>-0.00010544999800...</td><td>-0.00010218720356...</td><td>-0.00010491602545428</td><td>-0.00010486089568...</td><td>-0.00010590824777...</td><td>-0.00010321507601...</td><td>-0.00010275802070302</td><td>-0.00011087523222...</td><td>-0.00010305543062...</td><td>-0.00010489042033...</td><td>-0.00010530735344...</td><td>-0.00011466656096...</td><td>-0.00010053247847...</td><td>-0.00010022573314...</td><td>0</td><td>0</td></tr>\n",
       "<tr><td>max</td><td>99999</td><td>7.55406974741191e-05</td><td>9.99769856171626</td><td>9.67444968403876e-05</td><td>9.92501936512661</td><td>9.99846034625664e-05</td><td>9.91116576052911</td><td>9.97044721041161</td><td>9.90825458583455</td><td>9.96754672601408e-05</td><td>9.990175905094</td><td>9.93981974172569</td><td>9.98580333033583e-05</td><td>9.42995346153325e-05</td><td>9.42849461347088e-05</td><td>8.92751304924777e-05</td><td>9.62125212195424e-05</td><td>9.99380070915989e-05</td><td>9.72652650759061e-06</td><td>9.83848235865115e-06</td><td>9.94641486230186</td><td>9.97908705775956</td><td>9.8395225212225e-05</td><td>9.97423880847133</td><td>9.98525037704461e-05</td><td>9.7739127160934e-05</td><td>9.53790653973052e-05</td><td>9.94394238269967e-05</td><td>9.9271564256982e-06</td><td>999.9</td><td>1</td></tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "+-------+------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+-----------------+--------------------+\n",
       "|summary|              Time|                  V1|                  V2|                  V3|                  V4|                  V5|                  V6|                  V7|                  V8|                  V9|                 V10|                 V11|                 V12|                 V13|                 V14|                 V15|                 V16|                 V17|                 V18|                 V19|                 V20|                 V21|                 V22|                 V23|                 V24|                 V25|                 V26|                 V27|                 V28|           Amount|               Class|\n",
       "+-------+------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+-----------------+--------------------+\n",
       "|  count|            284807|              284807|              284807|              284807|              284807|              284807|              284807|              284807|              284807|              284807|              284807|              284807|              284807|              284807|              284807|              284807|              284807|              284807|              284807|              284807|              284807|              284807|              284807|              284807|              284807|              284807|              284807|              284807|              284807|           284807|              284807|\n",
       "|   mean| 94813.85957508067|2.235360406313924...|6.865749819392767...|-5.82471054445228...|2.011824365682531...|3.704311530463074...|1.140033807220101...|-1.14961392324716...|-2.9538691083434E-16|-2.08207854988096...|2.145945990061367...|-1.27734880360795...|-3.30833340134460...|1.034652530922445...|1.708454024825641...|5.933285192758958...|1.394705224939437...|-2.87403480811790...|1.749967860942900...|9.38053027649593E-16|3.81607955077877E-16|2.850084518050253...|-7.66409282164773...| 2.9538691083434E-16|4.454953538333312...|1.034652530922445...|1.808246900107513...|-3.52368642620288...|-1.13364706320206...| 88.3496192509521|0.001727485630620034|\n",
       "| stddev|47488.145954566324|  1.9586958038574904|  1.6513085794769997|  1.5162550051777732|   1.415868574940927|   1.380246734031437|  1.3322710897575714|  1.2370935981826632|  1.1943529026692048|  1.0986320892243222|  1.0888497654025215|  1.0207130277115581|  0.9992013895301415|  0.9952742301251558|  0.9585956112570617|  0.9153160116104389|  0.8762528873883704|    0.84933706367439|  0.8381762095288453|  0.8140405007685797|  0.7709250248871159|  0.7345240143713125|  0.7257015604409107|  0.6244602955949898|  0.6056470678271603|  0.5212780705409427| 0.48222701326105666|  0.4036324949650313| 0.33008326416025036|250.1201092401885|0.041527189635464985|\n",
       "|    min|                 0|-0.00012931370800...|-0.00010296722561...|-0.00010859127517...|-0.00011921826106...|-0.00010366562678...|-0.00010234903761...|-0.00010533581684...|-0.00010065655617...|-0.00010181309940...|-0.00010546343208...|-0.00010815155961...|-0.00010708756424...| -0.0001012565441717|-0.00011026584350728|-0.00010544999800...|-0.00010218720356...|-0.00010491602545428|-0.00010486089568...|-0.00010590824777...|-0.00010321507601...|-0.00010275802070302|-0.00011087523222...|-0.00010305543062...|-0.00010489042033...|-0.00010530735344...|-0.00011466656096...|-0.00010053247847...|-0.00010022573314...|                0|                   0|\n",
       "|    max|             99999|7.55406974741191e-05|    9.99769856171626|9.67444968403876e-05|    9.92501936512661|9.99846034625664e-05|    9.91116576052911|    9.97044721041161|    9.90825458583455|9.96754672601408e-05|      9.990175905094|    9.93981974172569|9.98580333033583e-05|9.42995346153325e-05|9.42849461347088e-05|8.92751304924777e-05|9.62125212195424e-05|9.99380070915989e-05|9.72652650759061e-06|9.83848235865115e-06|    9.94641486230186|    9.97908705775956| 9.8395225212225e-05|    9.97423880847133|9.98525037704461e-05| 9.7739127160934e-05|9.53790653973052e-05|9.94394238269967e-05| 9.9271564256982e-06|            999.9|                   1|\n",
       "+-------+------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+-----------------+--------------------+"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "284807"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dropna().count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Map RDD to DataFrame using Row\n",
    "Read <a href=\" https://www.dcc.fc.up.pt/~ltorgo/Regression/cal_housing.html\"> California housing dataset </a> into an RDD then map it to the spark DataFrame. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyspark as spark\n",
    "from pyspark.sql import SparkSession\n",
    "spark = SparkSession \\\n",
    "    .builder \\\n",
    "    .appName(\"Python Spark SQL basic example\") \\\n",
    "    .config(\"spark.some.config.option\", \"some-value\") \\\n",
    "    .getOrCreate()\n",
    "\n",
    "#For pandas like visualization - no need to use df.show()\n",
    "spark.conf.set(\"spark.sql.repl.eagerEval.enabled\", True) \n",
    "\n",
    "sc =  spark.sparkContext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['-122.230000',\n",
       "  '37.880000',\n",
       "  '41.000000',\n",
       "  '880.000000',\n",
       "  '129.000000',\n",
       "  '322.000000',\n",
       "  '126.000000',\n",
       "  '8.325200',\n",
       "  '452600.000000'],\n",
       " ['-122.220000',\n",
       "  '37.860000',\n",
       "  '21.000000',\n",
       "  '7099.000000',\n",
       "  '1106.000000',\n",
       "  '2401.000000',\n",
       "  '1138.000000',\n",
       "  '8.301400',\n",
       "  '358500.000000']]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rdd = sc.textFile('cal_housing.data')\n",
    "rdd = rdd.map(lambda line:line.split(\",\")) # split the lines\n",
    "#rdd.collect() # to print all lines\n",
    "rdd.take(2)    # to print only 2 first lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import Row\n",
    "df =rdd.map(lambda line: Row(longitude=line[0],\n",
    "                            lattitude=line[1],\n",
    "                            housingMedAge=line[2],\n",
    "                            totalRooms=line[3],\n",
    "                            totalBedR=line[4],\n",
    "                            population=line[5],\n",
    "                            households=line[6],\n",
    "                            medianIncome=line[7],\n",
    "                            medianHouseValue=line[8])).toDF()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['longitude',\n",
       " 'lattitude',\n",
       " 'housingMedAge',\n",
       " 'totalRooms',\n",
       " 'totalBedR',\n",
       " 'population',\n",
       " 'households',\n",
       " 'medianIncome',\n",
       " 'medianHouseValue']"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border='1'>\n",
       "<tr><th>summary</th><th>longitude</th><th>lattitude</th><th>housingMedAge</th><th>totalRooms</th><th>totalBedR</th><th>population</th><th>households</th><th>medianIncome</th><th>medianHouseValue</th></tr>\n",
       "<tr><td>count</td><td>20640</td><td>20640</td><td>20640</td><td>20640</td><td>20640</td><td>20640</td><td>20640</td><td>20640</td><td>20640</td></tr>\n",
       "<tr><td>mean</td><td>-119.5697044573641</td><td>35.631861434108586</td><td>28.639486434108527</td><td>2635.7630813953488</td><td>537.8980135658915</td><td>1425.4767441860465</td><td>499.5396802325581</td><td>3.870671002906978</td><td>206855.81690891474</td></tr>\n",
       "<tr><td>stddev</td><td>2.003531723502633</td><td>2.1359523974571233</td><td>12.585557612111613</td><td>2181.6152515827994</td><td>421.2479059431315</td><td>1132.4621217653385</td><td>382.32975283161136</td><td>1.8998217179452634</td><td>115395.61587441381</td></tr>\n",
       "<tr><td>min</td><td>-114.310000</td><td>32.540000</td><td>1.000000</td><td>100.000000</td><td>1.000000</td><td>100.000000</td><td>1.000000</td><td>0.499900</td><td>100000.000000</td></tr>\n",
       "<tr><td>max</td><td>-124.350000</td><td>41.950000</td><td>9.000000</td><td>9998.000000</td><td>999.000000</td><td>999.000000</td><td>999.000000</td><td>9.907100</td><td>99900.000000</td></tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "+-------+------------------+------------------+------------------+------------------+-----------------+------------------+------------------+------------------+------------------+\n",
       "|summary|         longitude|         lattitude|     housingMedAge|        totalRooms|        totalBedR|        population|        households|      medianIncome|  medianHouseValue|\n",
       "+-------+------------------+------------------+------------------+------------------+-----------------+------------------+------------------+------------------+------------------+\n",
       "|  count|             20640|             20640|             20640|             20640|            20640|             20640|             20640|             20640|             20640|\n",
       "|   mean|-119.5697044573641|35.631861434108586|28.639486434108527|2635.7630813953488|537.8980135658915|1425.4767441860465| 499.5396802325581| 3.870671002906978|206855.81690891474|\n",
       "| stddev| 2.003531723502633|2.1359523974571233|12.585557612111613|2181.6152515827994|421.2479059431315|1132.4621217653385|382.32975283161136|1.8998217179452634|115395.61587441381|\n",
       "|    min|       -114.310000|         32.540000|          1.000000|        100.000000|         1.000000|        100.000000|          1.000000|          0.499900|     100000.000000|\n",
       "|    max|       -124.350000|         41.950000|          9.000000|       9998.000000|       999.000000|        999.000000|        999.000000|          9.907100|      99900.000000|\n",
       "+-------+------------------+------------------+------------------+------------------+-----------------+------------------+------------------+------------------+------------------+"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border='1'>\n",
       "<tr><th>longitude</th><th>lattitude</th><th>housingMedAge</th><th>totalRooms</th><th>totalBedR</th><th>population</th><th>households</th><th>medianIncome</th><th>medianHouseValue</th></tr>\n",
       "<tr><td>-122.230000</td><td>37.880000</td><td>41.000000</td><td>880.000000</td><td>129.000000</td><td>322.000000</td><td>126.000000</td><td>8.325200</td><td>452600.000000</td></tr>\n",
       "<tr><td>-122.220000</td><td>37.860000</td><td>21.000000</td><td>7099.000000</td><td>1106.000000</td><td>2401.000000</td><td>1138.000000</td><td>8.301400</td><td>358500.000000</td></tr>\n",
       "<tr><td>-122.240000</td><td>37.850000</td><td>52.000000</td><td>1467.000000</td><td>190.000000</td><td>496.000000</td><td>177.000000</td><td>7.257400</td><td>352100.000000</td></tr>\n",
       "<tr><td>-122.250000</td><td>37.850000</td><td>52.000000</td><td>1274.000000</td><td>235.000000</td><td>558.000000</td><td>219.000000</td><td>5.643100</td><td>341300.000000</td></tr>\n",
       "<tr><td>-122.250000</td><td>37.850000</td><td>52.000000</td><td>1627.000000</td><td>280.000000</td><td>565.000000</td><td>259.000000</td><td>3.846200</td><td>342200.000000</td></tr>\n",
       "<tr><td>-122.250000</td><td>37.850000</td><td>52.000000</td><td>919.000000</td><td>213.000000</td><td>413.000000</td><td>193.000000</td><td>4.036800</td><td>269700.000000</td></tr>\n",
       "<tr><td>-122.250000</td><td>37.840000</td><td>52.000000</td><td>2535.000000</td><td>489.000000</td><td>1094.000000</td><td>514.000000</td><td>3.659100</td><td>299200.000000</td></tr>\n",
       "<tr><td>-122.250000</td><td>37.840000</td><td>52.000000</td><td>3104.000000</td><td>687.000000</td><td>1157.000000</td><td>647.000000</td><td>3.120000</td><td>241400.000000</td></tr>\n",
       "<tr><td>-122.260000</td><td>37.840000</td><td>42.000000</td><td>2555.000000</td><td>665.000000</td><td>1206.000000</td><td>595.000000</td><td>2.080400</td><td>226700.000000</td></tr>\n",
       "<tr><td>-122.250000</td><td>37.840000</td><td>52.000000</td><td>3549.000000</td><td>707.000000</td><td>1551.000000</td><td>714.000000</td><td>3.691200</td><td>261100.000000</td></tr>\n",
       "<tr><td>-122.260000</td><td>37.850000</td><td>52.000000</td><td>2202.000000</td><td>434.000000</td><td>910.000000</td><td>402.000000</td><td>3.203100</td><td>281500.000000</td></tr>\n",
       "<tr><td>-122.260000</td><td>37.850000</td><td>52.000000</td><td>3503.000000</td><td>752.000000</td><td>1504.000000</td><td>734.000000</td><td>3.270500</td><td>241800.000000</td></tr>\n",
       "<tr><td>-122.260000</td><td>37.850000</td><td>52.000000</td><td>2491.000000</td><td>474.000000</td><td>1098.000000</td><td>468.000000</td><td>3.075000</td><td>213500.000000</td></tr>\n",
       "<tr><td>-122.260000</td><td>37.840000</td><td>52.000000</td><td>696.000000</td><td>191.000000</td><td>345.000000</td><td>174.000000</td><td>2.673600</td><td>191300.000000</td></tr>\n",
       "<tr><td>-122.260000</td><td>37.850000</td><td>52.000000</td><td>2643.000000</td><td>626.000000</td><td>1212.000000</td><td>620.000000</td><td>1.916700</td><td>159200.000000</td></tr>\n",
       "<tr><td>-122.260000</td><td>37.850000</td><td>50.000000</td><td>1120.000000</td><td>283.000000</td><td>697.000000</td><td>264.000000</td><td>2.125000</td><td>140000.000000</td></tr>\n",
       "<tr><td>-122.270000</td><td>37.850000</td><td>52.000000</td><td>1966.000000</td><td>347.000000</td><td>793.000000</td><td>331.000000</td><td>2.775000</td><td>152500.000000</td></tr>\n",
       "<tr><td>-122.270000</td><td>37.850000</td><td>52.000000</td><td>1228.000000</td><td>293.000000</td><td>648.000000</td><td>303.000000</td><td>2.120200</td><td>155500.000000</td></tr>\n",
       "<tr><td>-122.260000</td><td>37.840000</td><td>50.000000</td><td>2239.000000</td><td>455.000000</td><td>990.000000</td><td>419.000000</td><td>1.991100</td><td>158700.000000</td></tr>\n",
       "<tr><td>-122.270000</td><td>37.840000</td><td>52.000000</td><td>1503.000000</td><td>298.000000</td><td>690.000000</td><td>275.000000</td><td>2.603300</td><td>162900.000000</td></tr>\n",
       "</table>\n",
       "only showing top 20 rows\n"
      ],
      "text/plain": [
       "+-----------+---------+-------------+-----------+-----------+-----------+-----------+------------+----------------+\n",
       "|  longitude|lattitude|housingMedAge| totalRooms|  totalBedR| population| households|medianIncome|medianHouseValue|\n",
       "+-----------+---------+-------------+-----------+-----------+-----------+-----------+------------+----------------+\n",
       "|-122.230000|37.880000|    41.000000| 880.000000| 129.000000| 322.000000| 126.000000|    8.325200|   452600.000000|\n",
       "|-122.220000|37.860000|    21.000000|7099.000000|1106.000000|2401.000000|1138.000000|    8.301400|   358500.000000|\n",
       "|-122.240000|37.850000|    52.000000|1467.000000| 190.000000| 496.000000| 177.000000|    7.257400|   352100.000000|\n",
       "|-122.250000|37.850000|    52.000000|1274.000000| 235.000000| 558.000000| 219.000000|    5.643100|   341300.000000|\n",
       "|-122.250000|37.850000|    52.000000|1627.000000| 280.000000| 565.000000| 259.000000|    3.846200|   342200.000000|\n",
       "|-122.250000|37.850000|    52.000000| 919.000000| 213.000000| 413.000000| 193.000000|    4.036800|   269700.000000|\n",
       "|-122.250000|37.840000|    52.000000|2535.000000| 489.000000|1094.000000| 514.000000|    3.659100|   299200.000000|\n",
       "|-122.250000|37.840000|    52.000000|3104.000000| 687.000000|1157.000000| 647.000000|    3.120000|   241400.000000|\n",
       "|-122.260000|37.840000|    42.000000|2555.000000| 665.000000|1206.000000| 595.000000|    2.080400|   226700.000000|\n",
       "|-122.250000|37.840000|    52.000000|3549.000000| 707.000000|1551.000000| 714.000000|    3.691200|   261100.000000|\n",
       "|-122.260000|37.850000|    52.000000|2202.000000| 434.000000| 910.000000| 402.000000|    3.203100|   281500.000000|\n",
       "|-122.260000|37.850000|    52.000000|3503.000000| 752.000000|1504.000000| 734.000000|    3.270500|   241800.000000|\n",
       "|-122.260000|37.850000|    52.000000|2491.000000| 474.000000|1098.000000| 468.000000|    3.075000|   213500.000000|\n",
       "|-122.260000|37.840000|    52.000000| 696.000000| 191.000000| 345.000000| 174.000000|    2.673600|   191300.000000|\n",
       "|-122.260000|37.850000|    52.000000|2643.000000| 626.000000|1212.000000| 620.000000|    1.916700|   159200.000000|\n",
       "|-122.260000|37.850000|    50.000000|1120.000000| 283.000000| 697.000000| 264.000000|    2.125000|   140000.000000|\n",
       "|-122.270000|37.850000|    52.000000|1966.000000| 347.000000| 793.000000| 331.000000|    2.775000|   152500.000000|\n",
       "|-122.270000|37.850000|    52.000000|1228.000000| 293.000000| 648.000000| 303.000000|    2.120200|   155500.000000|\n",
       "|-122.260000|37.840000|    50.000000|2239.000000| 455.000000| 990.000000| 419.000000|    1.991100|   158700.000000|\n",
       "|-122.270000|37.840000|    52.000000|1503.000000| 298.000000| 690.000000| 275.000000|    2.603300|   162900.000000|\n",
       "+-----------+---------+-------------+-----------+-----------+-----------+-----------+------------+----------------+\n",
       "only showing top 20 rows"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Access a column, adding a new column\n",
    "Remember that operations in spark DataFrame are **lazy** in nature. However, in pandas we get the result as soon as we apply any operation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Column<b'households'>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['households']\n",
    "df[6]\n",
    "df.households"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Add a new column to the df:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border='1'>\n",
       "<tr><th>longitude</th><th>lattitude</th><th>housingMedAge</th><th>totalRooms</th><th>totalBedR</th><th>population</th><th>households</th><th>medianIncome</th><th>medianHouseValue</th><th>housePop</th></tr>\n",
       "<tr><td>-122.230000</td><td>37.880000</td><td>41.000000</td><td>880.000000</td><td>129.000000</td><td>322.000000</td><td>126.000000</td><td>8.325200</td><td>452600.000000</td><td>0.391304347826087</td></tr>\n",
       "<tr><td>-122.220000</td><td>37.860000</td><td>21.000000</td><td>7099.000000</td><td>1106.000000</td><td>2401.000000</td><td>1138.000000</td><td>8.301400</td><td>358500.000000</td><td>0.4739691795085381</td></tr>\n",
       "<tr><td>-122.240000</td><td>37.850000</td><td>52.000000</td><td>1467.000000</td><td>190.000000</td><td>496.000000</td><td>177.000000</td><td>7.257400</td><td>352100.000000</td><td>0.35685483870967744</td></tr>\n",
       "<tr><td>-122.250000</td><td>37.850000</td><td>52.000000</td><td>1274.000000</td><td>235.000000</td><td>558.000000</td><td>219.000000</td><td>5.643100</td><td>341300.000000</td><td>0.3924731182795699</td></tr>\n",
       "<tr><td>-122.250000</td><td>37.850000</td><td>52.000000</td><td>1627.000000</td><td>280.000000</td><td>565.000000</td><td>259.000000</td><td>3.846200</td><td>342200.000000</td><td>0.4584070796460177</td></tr>\n",
       "<tr><td>-122.250000</td><td>37.850000</td><td>52.000000</td><td>919.000000</td><td>213.000000</td><td>413.000000</td><td>193.000000</td><td>4.036800</td><td>269700.000000</td><td>0.46731234866828086</td></tr>\n",
       "<tr><td>-122.250000</td><td>37.840000</td><td>52.000000</td><td>2535.000000</td><td>489.000000</td><td>1094.000000</td><td>514.000000</td><td>3.659100</td><td>299200.000000</td><td>0.46983546617915906</td></tr>\n",
       "<tr><td>-122.250000</td><td>37.840000</td><td>52.000000</td><td>3104.000000</td><td>687.000000</td><td>1157.000000</td><td>647.000000</td><td>3.120000</td><td>241400.000000</td><td>0.5592048401037165</td></tr>\n",
       "<tr><td>-122.260000</td><td>37.840000</td><td>42.000000</td><td>2555.000000</td><td>665.000000</td><td>1206.000000</td><td>595.000000</td><td>2.080400</td><td>226700.000000</td><td>0.49336650082918737</td></tr>\n",
       "<tr><td>-122.250000</td><td>37.840000</td><td>52.000000</td><td>3549.000000</td><td>707.000000</td><td>1551.000000</td><td>714.000000</td><td>3.691200</td><td>261100.000000</td><td>0.46034816247582205</td></tr>\n",
       "<tr><td>-122.260000</td><td>37.850000</td><td>52.000000</td><td>2202.000000</td><td>434.000000</td><td>910.000000</td><td>402.000000</td><td>3.203100</td><td>281500.000000</td><td>0.44175824175824174</td></tr>\n",
       "<tr><td>-122.260000</td><td>37.850000</td><td>52.000000</td><td>3503.000000</td><td>752.000000</td><td>1504.000000</td><td>734.000000</td><td>3.270500</td><td>241800.000000</td><td>0.488031914893617</td></tr>\n",
       "<tr><td>-122.260000</td><td>37.850000</td><td>52.000000</td><td>2491.000000</td><td>474.000000</td><td>1098.000000</td><td>468.000000</td><td>3.075000</td><td>213500.000000</td><td>0.4262295081967213</td></tr>\n",
       "<tr><td>-122.260000</td><td>37.840000</td><td>52.000000</td><td>696.000000</td><td>191.000000</td><td>345.000000</td><td>174.000000</td><td>2.673600</td><td>191300.000000</td><td>0.5043478260869565</td></tr>\n",
       "<tr><td>-122.260000</td><td>37.850000</td><td>52.000000</td><td>2643.000000</td><td>626.000000</td><td>1212.000000</td><td>620.000000</td><td>1.916700</td><td>159200.000000</td><td>0.5115511551155115</td></tr>\n",
       "<tr><td>-122.260000</td><td>37.850000</td><td>50.000000</td><td>1120.000000</td><td>283.000000</td><td>697.000000</td><td>264.000000</td><td>2.125000</td><td>140000.000000</td><td>0.3787661406025825</td></tr>\n",
       "<tr><td>-122.270000</td><td>37.850000</td><td>52.000000</td><td>1966.000000</td><td>347.000000</td><td>793.000000</td><td>331.000000</td><td>2.775000</td><td>152500.000000</td><td>0.41740226986128626</td></tr>\n",
       "<tr><td>-122.270000</td><td>37.850000</td><td>52.000000</td><td>1228.000000</td><td>293.000000</td><td>648.000000</td><td>303.000000</td><td>2.120200</td><td>155500.000000</td><td>0.4675925925925926</td></tr>\n",
       "<tr><td>-122.260000</td><td>37.840000</td><td>50.000000</td><td>2239.000000</td><td>455.000000</td><td>990.000000</td><td>419.000000</td><td>1.991100</td><td>158700.000000</td><td>0.42323232323232324</td></tr>\n",
       "<tr><td>-122.270000</td><td>37.840000</td><td>52.000000</td><td>1503.000000</td><td>298.000000</td><td>690.000000</td><td>275.000000</td><td>2.603300</td><td>162900.000000</td><td>0.39855072463768115</td></tr>\n",
       "</table>\n",
       "only showing top 20 rows\n"
      ],
      "text/plain": [
       "+-----------+---------+-------------+-----------+-----------+-----------+-----------+------------+----------------+-------------------+\n",
       "|  longitude|lattitude|housingMedAge| totalRooms|  totalBedR| population| households|medianIncome|medianHouseValue|           housePop|\n",
       "+-----------+---------+-------------+-----------+-----------+-----------+-----------+------------+----------------+-------------------+\n",
       "|-122.230000|37.880000|    41.000000| 880.000000| 129.000000| 322.000000| 126.000000|    8.325200|   452600.000000|  0.391304347826087|\n",
       "|-122.220000|37.860000|    21.000000|7099.000000|1106.000000|2401.000000|1138.000000|    8.301400|   358500.000000| 0.4739691795085381|\n",
       "|-122.240000|37.850000|    52.000000|1467.000000| 190.000000| 496.000000| 177.000000|    7.257400|   352100.000000|0.35685483870967744|\n",
       "|-122.250000|37.850000|    52.000000|1274.000000| 235.000000| 558.000000| 219.000000|    5.643100|   341300.000000| 0.3924731182795699|\n",
       "|-122.250000|37.850000|    52.000000|1627.000000| 280.000000| 565.000000| 259.000000|    3.846200|   342200.000000| 0.4584070796460177|\n",
       "|-122.250000|37.850000|    52.000000| 919.000000| 213.000000| 413.000000| 193.000000|    4.036800|   269700.000000|0.46731234866828086|\n",
       "|-122.250000|37.840000|    52.000000|2535.000000| 489.000000|1094.000000| 514.000000|    3.659100|   299200.000000|0.46983546617915906|\n",
       "|-122.250000|37.840000|    52.000000|3104.000000| 687.000000|1157.000000| 647.000000|    3.120000|   241400.000000| 0.5592048401037165|\n",
       "|-122.260000|37.840000|    42.000000|2555.000000| 665.000000|1206.000000| 595.000000|    2.080400|   226700.000000|0.49336650082918737|\n",
       "|-122.250000|37.840000|    52.000000|3549.000000| 707.000000|1551.000000| 714.000000|    3.691200|   261100.000000|0.46034816247582205|\n",
       "|-122.260000|37.850000|    52.000000|2202.000000| 434.000000| 910.000000| 402.000000|    3.203100|   281500.000000|0.44175824175824174|\n",
       "|-122.260000|37.850000|    52.000000|3503.000000| 752.000000|1504.000000| 734.000000|    3.270500|   241800.000000|  0.488031914893617|\n",
       "|-122.260000|37.850000|    52.000000|2491.000000| 474.000000|1098.000000| 468.000000|    3.075000|   213500.000000| 0.4262295081967213|\n",
       "|-122.260000|37.840000|    52.000000| 696.000000| 191.000000| 345.000000| 174.000000|    2.673600|   191300.000000| 0.5043478260869565|\n",
       "|-122.260000|37.850000|    52.000000|2643.000000| 626.000000|1212.000000| 620.000000|    1.916700|   159200.000000| 0.5115511551155115|\n",
       "|-122.260000|37.850000|    50.000000|1120.000000| 283.000000| 697.000000| 264.000000|    2.125000|   140000.000000| 0.3787661406025825|\n",
       "|-122.270000|37.850000|    52.000000|1966.000000| 347.000000| 793.000000| 331.000000|    2.775000|   152500.000000|0.41740226986128626|\n",
       "|-122.270000|37.850000|    52.000000|1228.000000| 293.000000| 648.000000| 303.000000|    2.120200|   155500.000000| 0.4675925925925926|\n",
       "|-122.260000|37.840000|    50.000000|2239.000000| 455.000000| 990.000000| 419.000000|    1.991100|   158700.000000|0.42323232323232324|\n",
       "|-122.270000|37.840000|    52.000000|1503.000000| 298.000000| 690.000000| 275.000000|    2.603300|   162900.000000|0.39855072463768115|\n",
       "+-----------+---------+-------------+-----------+-----------+-----------+-----------+------------+----------------+-------------------+\n",
       "only showing top 20 rows"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.withColumn('housePop', df.households/df.population)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border='1'>\n",
       "<tr><th>households</th><th>population</th></tr>\n",
       "<tr><td>126.000000</td><td>322.000000</td></tr>\n",
       "<tr><td>1138.000000</td><td>2401.000000</td></tr>\n",
       "<tr><td>177.000000</td><td>496.000000</td></tr>\n",
       "<tr><td>219.000000</td><td>558.000000</td></tr>\n",
       "<tr><td>259.000000</td><td>565.000000</td></tr>\n",
       "<tr><td>193.000000</td><td>413.000000</td></tr>\n",
       "<tr><td>514.000000</td><td>1094.000000</td></tr>\n",
       "<tr><td>647.000000</td><td>1157.000000</td></tr>\n",
       "<tr><td>595.000000</td><td>1206.000000</td></tr>\n",
       "<tr><td>714.000000</td><td>1551.000000</td></tr>\n",
       "<tr><td>402.000000</td><td>910.000000</td></tr>\n",
       "<tr><td>734.000000</td><td>1504.000000</td></tr>\n",
       "<tr><td>468.000000</td><td>1098.000000</td></tr>\n",
       "<tr><td>174.000000</td><td>345.000000</td></tr>\n",
       "<tr><td>620.000000</td><td>1212.000000</td></tr>\n",
       "<tr><td>264.000000</td><td>697.000000</td></tr>\n",
       "<tr><td>331.000000</td><td>793.000000</td></tr>\n",
       "<tr><td>303.000000</td><td>648.000000</td></tr>\n",
       "<tr><td>419.000000</td><td>990.000000</td></tr>\n",
       "<tr><td>275.000000</td><td>690.000000</td></tr>\n",
       "</table>\n",
       "only showing top 20 rows\n"
      ],
      "text/plain": [
       "+-----------+-----------+\n",
       "| households| population|\n",
       "+-----------+-----------+\n",
       "| 126.000000| 322.000000|\n",
       "|1138.000000|2401.000000|\n",
       "| 177.000000| 496.000000|\n",
       "| 219.000000| 558.000000|\n",
       "| 259.000000| 565.000000|\n",
       "| 193.000000| 413.000000|\n",
       "| 514.000000|1094.000000|\n",
       "| 647.000000|1157.000000|\n",
       "| 595.000000|1206.000000|\n",
       "| 714.000000|1551.000000|\n",
       "| 402.000000| 910.000000|\n",
       "| 734.000000|1504.000000|\n",
       "| 468.000000|1098.000000|\n",
       "| 174.000000| 345.000000|\n",
       "| 620.000000|1212.000000|\n",
       "| 264.000000| 697.000000|\n",
       "| 331.000000| 793.000000|\n",
       "| 303.000000| 648.000000|\n",
       "| 419.000000| 990.000000|\n",
       "| 275.000000| 690.000000|\n",
       "+-----------+-----------+\n",
       "only showing top 20 rows"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.select('households','population')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}