SparkTorch_YARN_Example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Set Python - Spark environment.\n",
    "import os\n",
    "import sys\n",
    "#Setting the path for Spark\n",
    "# os.environ['SPARK_HOME'] = \"/home/Kunal/Downloads/spark-2.4.7-bin-hadoop2.7\"\n",
    "os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages org.apache.spark:spark-sql-kafka-0-10_2.11:2.4.7,org.apache.spark:spark-streaming-kafka-0-10_2.11:2.4.7 pyspark-shell'\n",
    "os.environ['PYSPARK_PYTHON'] = sys.executable\n",
    "os.environ['PYSPARK_DRIVER_PYTHON'] = sys.executable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import packages\n",
    "from sparktorch import serialize_torch_obj, SparkTorch\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from pyspark.ml.feature import VectorAssembler\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.ml.pipeline import Pipeline\n",
    "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
    "from sparktorch import SparkTorch, serialize_torch_obj_lazy\n",
    "from pyspark.sql.functions import rand\n",
    "## Create SparkContext, SparkSession\n",
    "from pyspark.sql import *\n",
    "import pyspark\n",
    "from pyspark.conf import SparkConf\n",
    "from pyspark import SparkContext\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.sql.types import *\n",
    "from pyspark.sql.functions import col, udf, column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#setting configuration\n",
    "conf = SparkConf().setAppName(\"Python_Spark_example\").setMaster('yarn-client').set(\"spark.executor.memory\",\"5g\")\n",
    "sc = SparkContext(conf=conf)\n",
    "# sc=SparkContext.getOrCreate(conf=create_spark_conf().setMaster(\"local[4]\").set(\"spark.driver.memory\",\"8g\").set(\"spark.executor.memory\", '8g').set('spark.executor.cores', 4))\n",
    "sc.setLogLevel(\"ERROR\")\n",
    "# spark = SparkSession(sc)\n",
    "spark = SparkSession \\\n",
    "            .builder \\\n",
    "            .appName(\"Python Spark SQL basic example\") \\\n",
    "            .config(\"spark.sql.catalogImplementation=hive\").enableHiveSupport() \\\n",
    "            .getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# spark = SparkSession.builder.appName(\"examples\").master('yarn-client').getOrCreate()\n",
    "df = spark.read.option(\"inferSchema\", \"true\").csv('hdfs://localhost:9000//user/kunal/examples/mnist_train.csv').coalesce(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Typecast features into double\n",
    "for col_name in df.columns:\n",
    "    df = df.withColumn(col_name, col(col_name).cast('Double'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Filter null value\n",
    "df=df.filter(\"_c13 is not null\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "60000"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#number of record\n",
    "df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- _c0: double (nullable = true)\n",
      " |-- _c1: double (nullable = true)\n",
      " |-- _c2: double (nullable = true)\n",
      " |-- _c3: double (nullable = true)\n",
      " |-- _c4: double (nullable = true)\n",
      " |-- _c5: double (nullable = true)\n",
      " |-- _c6: double (nullable = true)\n",
      " |-- _c7: double (nullable = true)\n",
      " |-- _c8: double (nullable = true)\n",
      " |-- _c9: double (nullable = true)\n",
      " |-- _c10: double (nullable = true)\n",
      " |-- _c11: double (nullable = true)\n",
      " |-- _c12: double (nullable = true)\n",
      " |-- _c13: double (nullable = true)\n",
      " |-- _c14: double (nullable = true)\n",
      " |-- _c15: double (nullable = true)\n",
      " |-- _c16: double (nullable = true)\n",
      " |-- _c17: double (nullable = true)\n",
      " |-- _c18: double (nullable = true)\n",
      " |-- _c19: double (nullable = true)\n",
      " |-- _c20: double (nullable = true)\n",
      " |-- _c21: double (nullable = true)\n",
      " |-- _c22: double (nullable = true)\n",
      " |-- _c23: double (nullable = true)\n",
      " |-- _c24: double (nullable = true)\n",
      " |-- _c25: double (nullable = true)\n",
      " |-- _c26: double (nullable = true)\n",
      " |-- _c27: double (nullable = true)\n",
      " |-- _c28: double (nullable = true)\n",
      " |-- _c29: double (nullable = true)\n",
      " |-- _c30: double (nullable = true)\n",
      " |-- _c31: double (nullable = true)\n",
      " |-- _c32: double (nullable = true)\n",
      " |-- _c33: double (nullable = true)\n",
      " |-- _c34: double (nullable = true)\n",
      " |-- _c35: double (nullable = true)\n",
      " |-- _c36: double (nullable = true)\n",
      " |-- _c37: double (nullable = true)\n",
      " |-- _c38: double (nullable = true)\n",
      " |-- _c39: double (nullable = true)\n",
      " |-- _c40: double (nullable = true)\n",
      " |-- _c41: double (nullable = true)\n",
      " |-- _c42: double (nullable = true)\n",
      " |-- _c43: double (nullable = true)\n",
      " |-- _c44: double (nullable = true)\n",
      " |-- _c45: double (nullable = true)\n",
      " |-- _c46: double (nullable = true)\n",
      " |-- _c47: double (nullable = true)\n",
      " |-- _c48: double (nullable = true)\n",
      " |-- _c49: double (nullable = true)\n",
      " |-- _c50: double (nullable = true)\n",
      " |-- _c51: double (nullable = true)\n",
      " |-- _c52: double (nullable = true)\n",
      " |-- _c53: double (nullable = true)\n",
      " |-- _c54: double (nullable = true)\n",
      " |-- _c55: double (nullable = true)\n",
      " |-- _c56: double (nullable = true)\n",
      " |-- _c57: double (nullable = true)\n",
      " |-- _c58: double (nullable = true)\n",
      " |-- _c59: double (nullable = true)\n",
      " |-- _c60: double (nullable = true)\n",
      " |-- _c61: double (nullable = true)\n",
      " |-- _c62: double (nullable = true)\n",
      " |-- _c63: double (nullable = true)\n",
      " |-- _c64: double (nullable = true)\n",
      " |-- _c65: double (nullable = true)\n",
      " |-- _c66: double (nullable = true)\n",
      " |-- _c67: double (nullable = true)\n",
      " |-- _c68: double (nullable = true)\n",
      " |-- _c69: double (nullable = true)\n",
      " |-- _c70: double (nullable = true)\n",
      " |-- _c71: double (nullable = true)\n",
      " |-- _c72: double (nullable = true)\n",
      " |-- _c73: double (nullable = true)\n",
      " |-- _c74: double (nullable = true)\n",
      " |-- _c75: double (nullable = true)\n",
      " |-- _c76: double (nullable = true)\n",
      " |-- _c77: double (nullable = true)\n",
      " |-- _c78: double (nullable = true)\n",
      " |-- _c79: double (nullable = true)\n",
      " |-- _c80: double (nullable = true)\n",
      " |-- _c81: double (nullable = true)\n",
      " |-- _c82: double (nullable = true)\n",
      " |-- _c83: double (nullable = true)\n",
      " |-- _c84: double (nullable = true)\n",
      " |-- _c85: double (nullable = true)\n",
      " |-- _c86: double (nullable = true)\n",
      " |-- _c87: double (nullable = true)\n",
      " |-- _c88: double (nullable = true)\n",
      " |-- _c89: double (nullable = true)\n",
      " |-- _c90: double (nullable = true)\n",
      " |-- _c91: double (nullable = true)\n",
      " |-- _c92: double (nullable = true)\n",
      " |-- _c93: double (nullable = true)\n",
      " |-- _c94: double (nullable = true)\n",
      " |-- _c95: double (nullable = true)\n",
      " |-- _c96: double (nullable = true)\n",
      " |-- _c97: double (nullable = true)\n",
      " |-- _c98: double (nullable = true)\n",
      " |-- _c99: double (nullable = true)\n",
      " |-- _c100: double (nullable = true)\n",
      " |-- _c101: double (nullable = true)\n",
      " |-- _c102: double (nullable = true)\n",
      " |-- _c103: double (nullable = true)\n",
      " |-- _c104: double (nullable = true)\n",
      " |-- _c105: double (nullable = true)\n",
      " |-- _c106: double (nullable = true)\n",
      " |-- _c107: double (nullable = true)\n",
      " |-- _c108: double (nullable = true)\n",
      " |-- _c109: double (nullable = true)\n",
      " |-- _c110: double (nullable = true)\n",
      " |-- _c111: double (nullable = true)\n",
      " |-- _c112: double (nullable = true)\n",
      " |-- _c113: double (nullable = true)\n",
      " |-- _c114: double (nullable = true)\n",
      " |-- _c115: double (nullable = true)\n",
      " |-- _c116: double (nullable = true)\n",
      " |-- _c117: double (nullable = true)\n",
      " |-- _c118: double (nullable = true)\n",
      " |-- _c119: double (nullable = true)\n",
      " |-- _c120: double (nullable = true)\n",
      " |-- _c121: double (nullable = true)\n",
      " |-- _c122: double (nullable = true)\n",
      " |-- _c123: double (nullable = true)\n",
      " |-- _c124: double (nullable = true)\n",
      " |-- _c125: double (nullable = true)\n",
      " |-- _c126: double (nullable = true)\n",
      " |-- _c127: double (nullable = true)\n",
      " |-- _c128: double (nullable = true)\n",
      " |-- _c129: double (nullable = true)\n",
      " |-- _c130: double (nullable = true)\n",
      " |-- _c131: double (nullable = true)\n",
      " |-- _c132: double (nullable = true)\n",
      " |-- _c133: double (nullable = true)\n",
      " |-- _c134: double (nullable = true)\n",
      " |-- _c135: double (nullable = true)\n",
      " |-- _c136: double (nullable = true)\n",
      " |-- _c137: double (nullable = true)\n",
      " |-- _c138: double (nullable = true)\n",
      " |-- _c139: double (nullable = true)\n",
      " |-- _c140: double (nullable = true)\n",
      " |-- _c141: double (nullable = true)\n",
      " |-- _c142: double (nullable = true)\n",
      " |-- _c143: double (nullable = true)\n",
      " |-- _c144: double (nullable = true)\n",
      " |-- _c145: double (nullable = true)\n",
      " |-- _c146: double (nullable = true)\n",
      " |-- _c147: double (nullable = true)\n",
      " |-- _c148: double (nullable = true)\n",
      " |-- _c149: double (nullable = true)\n",
      " |-- _c150: double (nullable = true)\n",
      " |-- _c151: double (nullable = true)\n",
      " |-- _c152: double (nullable = true)\n",
      " |-- _c153: double (nullable = true)\n",
      " |-- _c154: double (nullable = true)\n",
      " |-- _c155: double (nullable = true)\n",
      " |-- _c156: double (nullable = true)\n",
      " |-- _c157: double (nullable = true)\n",
      " |-- _c158: double (nullable = true)\n",
      " |-- _c159: double (nullable = true)\n",
      " |-- _c160: double (nullable = true)\n",
      " |-- _c161: double (nullable = true)\n",
      " |-- _c162: double (nullable = true)\n",
      " |-- _c163: double (nullable = true)\n",
      " |-- _c164: double (nullable = true)\n",
      " |-- _c165: double (nullable = true)\n",
      " |-- _c166: double (nullable = true)\n",
      " |-- _c167: double (nullable = true)\n",
      " |-- _c168: double (nullable = true)\n",
      " |-- _c169: double (nullable = true)\n",
      " |-- _c170: double (nullable = true)\n",
      " |-- _c171: double (nullable = true)\n",
      " |-- _c172: double (nullable = true)\n",
      " |-- _c173: double (nullable = true)\n",
      " |-- _c174: double (nullable = true)\n",
      " |-- _c175: double (nullable = true)\n",
      " |-- _c176: double (nullable = true)\n",
      " |-- _c177: double (nullable = true)\n",
      " |-- _c178: double (nullable = true)\n",
      " |-- _c179: double (nullable = true)\n",
      " |-- _c180: double (nullable = true)\n",
      " |-- _c181: double (nullable = true)\n",
      " |-- _c182: double (nullable = true)\n",
      " |-- _c183: double (nullable = true)\n",
      " |-- _c184: double (nullable = true)\n",
      " |-- _c185: double (nullable = true)\n",
      " |-- _c186: double (nullable = true)\n",
      " |-- _c187: double (nullable = true)\n",
      " |-- _c188: double (nullable = true)\n",
      " |-- _c189: double (nullable = true)\n",
      " |-- _c190: double (nullable = true)\n",
      " |-- _c191: double (nullable = true)\n",
      " |-- _c192: double (nullable = true)\n",
      " |-- _c193: double (nullable = true)\n",
      " |-- _c194: double (nullable = true)\n",
      " |-- _c195: double (nullable = true)\n",
      " |-- _c196: double (nullable = true)\n",
      " |-- _c197: double (nullable = true)\n",
      " |-- _c198: double (nullable = true)\n",
      " |-- _c199: double (nullable = true)\n",
      " |-- _c200: double (nullable = true)\n",
      " |-- _c201: double (nullable = true)\n",
      " |-- _c202: double (nullable = true)\n",
      " |-- _c203: double (nullable = true)\n",
      " |-- _c204: double (nullable = true)\n",
      " |-- _c205: double (nullable = true)\n",
      " |-- _c206: double (nullable = true)\n",
      " |-- _c207: double (nullable = true)\n",
      " |-- _c208: double (nullable = true)\n",
      " |-- _c209: double (nullable = true)\n",
      " |-- _c210: double (nullable = true)\n",
      " |-- _c211: double (nullable = true)\n",
      " |-- _c212: double (nullable = true)\n",
      " |-- _c213: double (nullable = true)\n",
      " |-- _c214: double (nullable = true)\n",
      " |-- _c215: double (nullable = true)\n",
      " |-- _c216: double (nullable = true)\n",
      " |-- _c217: double (nullable = true)\n",
      " |-- _c218: double (nullable = true)\n",
      " |-- _c219: double (nullable = true)\n",
      " |-- _c220: double (nullable = true)\n",
      " |-- _c221: double (nullable = true)\n",
      " |-- _c222: double (nullable = true)\n",
      " |-- _c223: double (nullable = true)\n",
      " |-- _c224: double (nullable = true)\n",
      " |-- _c225: double (nullable = true)\n",
      " |-- _c226: double (nullable = true)\n",
      " |-- _c227: double (nullable = true)\n",
      " |-- _c228: double (nullable = true)\n",
      " |-- _c229: double (nullable = true)\n",
      " |-- _c230: double (nullable = true)\n",
      " |-- _c231: double (nullable = true)\n",
      " |-- _c232: double (nullable = true)\n",
      " |-- _c233: double (nullable = true)\n",
      " |-- _c234: double (nullable = true)\n",
      " |-- _c235: double (nullable = true)\n",
      " |-- _c236: double (nullable = true)\n",
      " |-- _c237: double (nullable = true)\n",
      " |-- _c238: double (nullable = true)\n",
      " |-- _c239: double (nullable = true)\n",
      " |-- _c240: double (nullable = true)\n",
      " |-- _c241: double (nullable = true)\n",
      " |-- _c242: double (nullable = true)\n",
      " |-- _c243: double (nullable = true)\n",
      " |-- _c244: double (nullable = true)\n",
      " |-- _c245: double (nullable = true)\n",
      " |-- _c246: double (nullable = true)\n",
      " |-- _c247: double (nullable = true)\n",
      " |-- _c248: double (nullable = true)\n",
      " |-- _c249: double (nullable = true)\n",
      " |-- _c250: double (nullable = true)\n",
      " |-- _c251: double (nullable = true)\n",
      " |-- _c252: double (nullable = true)\n",
      " |-- _c253: double (nullable = true)\n",
      " |-- _c254: double (nullable = true)\n",
      " |-- _c255: double (nullable = true)\n",
      " |-- _c256: double (nullable = true)\n",
      " |-- _c257: double (nullable = true)\n",
      " |-- _c258: double (nullable = true)\n",
      " |-- _c259: double (nullable = true)\n",
      " |-- _c260: double (nullable = true)\n",
      " |-- _c261: double (nullable = true)\n",
      " |-- _c262: double (nullable = true)\n",
      " |-- _c263: double (nullable = true)\n",
      " |-- _c264: double (nullable = true)\n",
      " |-- _c265: double (nullable = true)\n",
      " |-- _c266: double (nullable = true)\n",
      " |-- _c267: double (nullable = true)\n",
      " |-- _c268: double (nullable = true)\n",
      " |-- _c269: double (nullable = true)\n",
      " |-- _c270: double (nullable = true)\n",
      " |-- _c271: double (nullable = true)\n",
      " |-- _c272: double (nullable = true)\n",
      " |-- _c273: double (nullable = true)\n",
      " |-- _c274: double (nullable = true)\n",
      " |-- _c275: double (nullable = true)\n",
      " |-- _c276: double (nullable = true)\n",
      " |-- _c277: double (nullable = true)\n",
      " |-- _c278: double (nullable = true)\n",
      " |-- _c279: double (nullable = true)\n",
      " |-- _c280: double (nullable = true)\n",
      " |-- _c281: double (nullable = true)\n",
      " |-- _c282: double (nullable = true)\n",
      " |-- _c283: double (nullable = true)\n",
      " |-- _c284: double (nullable = true)\n",
      " |-- _c285: double (nullable = true)\n",
      " |-- _c286: double (nullable = true)\n",
      " |-- _c287: double (nullable = true)\n",
      " |-- _c288: double (nullable = true)\n",
      " |-- _c289: double (nullable = true)\n",
      " |-- _c290: double (nullable = true)\n",
      " |-- _c291: double (nullable = true)\n",
      " |-- _c292: double (nullable = true)\n",
      " |-- _c293: double (nullable = true)\n",
      " |-- _c294: double (nullable = true)\n",
      " |-- _c295: double (nullable = true)\n",
      " |-- _c296: double (nullable = true)\n",
      " |-- _c297: double (nullable = true)\n",
      " |-- _c298: double (nullable = true)\n",
      " |-- _c299: double (nullable = true)\n",
      " |-- _c300: double (nullable = true)\n",
      " |-- _c301: double (nullable = true)\n",
      " |-- _c302: double (nullable = true)\n",
      " |-- _c303: double (nullable = true)\n",
      " |-- _c304: double (nullable = true)\n",
      " |-- _c305: double (nullable = true)\n",
      " |-- _c306: double (nullable = true)\n",
      " |-- _c307: double (nullable = true)\n",
      " |-- _c308: double (nullable = true)\n",
      " |-- _c309: double (nullable = true)\n",
      " |-- _c310: double (nullable = true)\n",
      " |-- _c311: double (nullable = true)\n",
      " |-- _c312: double (nullable = true)\n",
      " |-- _c313: double (nullable = true)\n",
      " |-- _c314: double (nullable = true)\n",
      " |-- _c315: double (nullable = true)\n",
      " |-- _c316: double (nullable = true)\n",
      " |-- _c317: double (nullable = true)\n",
      " |-- _c318: double (nullable = true)\n",
      " |-- _c319: double (nullable = true)\n",
      " |-- _c320: double (nullable = true)\n",
      " |-- _c321: double (nullable = true)\n",
      " |-- _c322: double (nullable = true)\n",
      " |-- _c323: double (nullable = true)\n",
      " |-- _c324: double (nullable = true)\n",
      " |-- _c325: double (nullable = true)\n",
      " |-- _c326: double (nullable = true)\n",
      " |-- _c327: double (nullable = true)\n",
      " |-- _c328: double (nullable = true)\n",
      " |-- _c329: double (nullable = true)\n",
      " |-- _c330: double (nullable = true)\n",
      " |-- _c331: double (nullable = true)\n",
      " |-- _c332: double (nullable = true)\n",
      " |-- _c333: double (nullable = true)\n",
      " |-- _c334: double (nullable = true)\n",
      " |-- _c335: double (nullable = true)\n",
      " |-- _c336: double (nullable = true)\n",
      " |-- _c337: double (nullable = true)\n",
      " |-- _c338: double (nullable = true)\n",
      " |-- _c339: double (nullable = true)\n",
      " |-- _c340: double (nullable = true)\n",
      " |-- _c341: double (nullable = true)\n",
      " |-- _c342: double (nullable = true)\n",
      " |-- _c343: double (nullable = true)\n",
      " |-- _c344: double (nullable = true)\n",
      " |-- _c345: double (nullable = true)\n",
      " |-- _c346: double (nullable = true)\n",
      " |-- _c347: double (nullable = true)\n",
      " |-- _c348: double (nullable = true)\n",
      " |-- _c349: double (nullable = true)\n",
      " |-- _c350: double (nullable = true)\n",
      " |-- _c351: double (nullable = true)\n",
      " |-- _c352: double (nullable = true)\n",
      " |-- _c353: double (nullable = true)\n",
      " |-- _c354: double (nullable = true)\n",
      " |-- _c355: double (nullable = true)\n",
      " |-- _c356: double (nullable = true)\n",
      " |-- _c357: double (nullable = true)\n",
      " |-- _c358: double (nullable = true)\n",
      " |-- _c359: double (nullable = true)\n",
      " |-- _c360: double (nullable = true)\n",
      " |-- _c361: double (nullable = true)\n",
      " |-- _c362: double (nullable = true)\n",
      " |-- _c363: double (nullable = true)\n",
      " |-- _c364: double (nullable = true)\n",
      " |-- _c365: double (nullable = true)\n",
      " |-- _c366: double (nullable = true)\n",
      " |-- _c367: double (nullable = true)\n",
      " |-- _c368: double (nullable = true)\n",
      " |-- _c369: double (nullable = true)\n",
      " |-- _c370: double (nullable = true)\n",
      " |-- _c371: double (nullable = true)\n",
      " |-- _c372: double (nullable = true)\n",
      " |-- _c373: double (nullable = true)\n",
      " |-- _c374: double (nullable = true)\n",
      " |-- _c375: double (nullable = true)\n",
      " |-- _c376: double (nullable = true)\n",
      " |-- _c377: double (nullable = true)\n",
      " |-- _c378: double (nullable = true)\n",
      " |-- _c379: double (nullable = true)\n",
      " |-- _c380: double (nullable = true)\n",
      " |-- _c381: double (nullable = true)\n",
      " |-- _c382: double (nullable = true)\n",
      " |-- _c383: double (nullable = true)\n",
      " |-- _c384: double (nullable = true)\n",
      " |-- _c385: double (nullable = true)\n",
      " |-- _c386: double (nullable = true)\n",
      " |-- _c387: double (nullable = true)\n",
      " |-- _c388: double (nullable = true)\n",
      " |-- _c389: double (nullable = true)\n",
      " |-- _c390: double (nullable = true)\n",
      " |-- _c391: double (nullable = true)\n",
      " |-- _c392: double (nullable = true)\n",
      " |-- _c393: double (nullable = true)\n",
      " |-- _c394: double (nullable = true)\n",
      " |-- _c395: double (nullable = true)\n",
      " |-- _c396: double (nullable = true)\n",
      " |-- _c397: double (nullable = true)\n",
      " |-- _c398: double (nullable = true)\n",
      " |-- _c399: double (nullable = true)\n",
      " |-- _c400: double (nullable = true)\n",
      " |-- _c401: double (nullable = true)\n",
      " |-- _c402: double (nullable = true)\n",
      " |-- _c403: double (nullable = true)\n",
      " |-- _c404: double (nullable = true)\n",
      " |-- _c405: double (nullable = true)\n",
      " |-- _c406: double (nullable = true)\n",
      " |-- _c407: double (nullable = true)\n",
      " |-- _c408: double (nullable = true)\n",
      " |-- _c409: double (nullable = true)\n",
      " |-- _c410: double (nullable = true)\n",
      " |-- _c411: double (nullable = true)\n",
      " |-- _c412: double (nullable = true)\n",
      " |-- _c413: double (nullable = true)\n",
      " |-- _c414: double (nullable = true)\n",
      " |-- _c415: double (nullable = true)\n",
      " |-- _c416: double (nullable = true)\n",
      " |-- _c417: double (nullable = true)\n",
      " |-- _c418: double (nullable = true)\n",
      " |-- _c419: double (nullable = true)\n",
      " |-- _c420: double (nullable = true)\n",
      " |-- _c421: double (nullable = true)\n",
      " |-- _c422: double (nullable = true)\n",
      " |-- _c423: double (nullable = true)\n",
      " |-- _c424: double (nullable = true)\n",
      " |-- _c425: double (nullable = true)\n",
      " |-- _c426: double (nullable = true)\n",
      " |-- _c427: double (nullable = true)\n",
      " |-- _c428: double (nullable = true)\n",
      " |-- _c429: double (nullable = true)\n",
      " |-- _c430: double (nullable = true)\n",
      " |-- _c431: double (nullable = true)\n",
      " |-- _c432: double (nullable = true)\n",
      " |-- _c433: double (nullable = true)\n",
      " |-- _c434: double (nullable = true)\n",
      " |-- _c435: double (nullable = true)\n",
      " |-- _c436: double (nullable = true)\n",
      " |-- _c437: double (nullable = true)\n",
      " |-- _c438: double (nullable = true)\n",
      " |-- _c439: double (nullable = true)\n",
      " |-- _c440: double (nullable = true)\n",
      " |-- _c441: double (nullable = true)\n",
      " |-- _c442: double (nullable = true)\n",
      " |-- _c443: double (nullable = true)\n",
      " |-- _c444: double (nullable = true)\n",
      " |-- _c445: double (nullable = true)\n",
      " |-- _c446: double (nullable = true)\n",
      " |-- _c447: double (nullable = true)\n",
      " |-- _c448: double (nullable = true)\n",
      " |-- _c449: double (nullable = true)\n",
      " |-- _c450: double (nullable = true)\n",
      " |-- _c451: double (nullable = true)\n",
      " |-- _c452: double (nullable = true)\n",
      " |-- _c453: double (nullable = true)\n",
      " |-- _c454: double (nullable = true)\n",
      " |-- _c455: double (nullable = true)\n",
      " |-- _c456: double (nullable = true)\n",
      " |-- _c457: double (nullable = true)\n",
      " |-- _c458: double (nullable = true)\n",
      " |-- _c459: double (nullable = true)\n",
      " |-- _c460: double (nullable = true)\n",
      " |-- _c461: double (nullable = true)\n",
      " |-- _c462: double (nullable = true)\n",
      " |-- _c463: double (nullable = true)\n",
      " |-- _c464: double (nullable = true)\n",
      " |-- _c465: double (nullable = true)\n",
      " |-- _c466: double (nullable = true)\n",
      " |-- _c467: double (nullable = true)\n",
      " |-- _c468: double (nullable = true)\n",
      " |-- _c469: double (nullable = true)\n",
      " |-- _c470: double (nullable = true)\n",
      " |-- _c471: double (nullable = true)\n",
      " |-- _c472: double (nullable = true)\n",
      " |-- _c473: double (nullable = true)\n",
      " |-- _c474: double (nullable = true)\n",
      " |-- _c475: double (nullable = true)\n",
      " |-- _c476: double (nullable = true)\n",
      " |-- _c477: double (nullable = true)\n",
      " |-- _c478: double (nullable = true)\n",
      " |-- _c479: double (nullable = true)\n",
      " |-- _c480: double (nullable = true)\n",
      " |-- _c481: double (nullable = true)\n",
      " |-- _c482: double (nullable = true)\n",
      " |-- _c483: double (nullable = true)\n",
      " |-- _c484: double (nullable = true)\n",
      " |-- _c485: double (nullable = true)\n",
      " |-- _c486: double (nullable = true)\n",
      " |-- _c487: double (nullable = true)\n",
      " |-- _c488: double (nullable = true)\n",
      " |-- _c489: double (nullable = true)\n",
      " |-- _c490: double (nullable = true)\n",
      " |-- _c491: double (nullable = true)\n",
      " |-- _c492: double (nullable = true)\n",
      " |-- _c493: double (nullable = true)\n",
      " |-- _c494: double (nullable = true)\n",
      " |-- _c495: double (nullable = true)\n",
      " |-- _c496: double (nullable = true)\n",
      " |-- _c497: double (nullable = true)\n",
      " |-- _c498: double (nullable = true)\n",
      " |-- _c499: double (nullable = true)\n",
      " |-- _c500: double (nullable = true)\n",
      " |-- _c501: double (nullable = true)\n",
      " |-- _c502: double (nullable = true)\n",
      " |-- _c503: double (nullable = true)\n",
      " |-- _c504: double (nullable = true)\n",
      " |-- _c505: double (nullable = true)\n",
      " |-- _c506: double (nullable = true)\n",
      " |-- _c507: double (nullable = true)\n",
      " |-- _c508: double (nullable = true)\n",
      " |-- _c509: double (nullable = true)\n",
      " |-- _c510: double (nullable = true)\n",
      " |-- _c511: double (nullable = true)\n",
      " |-- _c512: double (nullable = true)\n",
      " |-- _c513: double (nullable = true)\n",
      " |-- _c514: double (nullable = true)\n",
      " |-- _c515: double (nullable = true)\n",
      " |-- _c516: double (nullable = true)\n",
      " |-- _c517: double (nullable = true)\n",
      " |-- _c518: double (nullable = true)\n",
      " |-- _c519: double (nullable = true)\n",
      " |-- _c520: double (nullable = true)\n",
      " |-- _c521: double (nullable = true)\n",
      " |-- _c522: double (nullable = true)\n",
      " |-- _c523: double (nullable = true)\n",
      " |-- _c524: double (nullable = true)\n",
      " |-- _c525: double (nullable = true)\n",
      " |-- _c526: double (nullable = true)\n",
      " |-- _c527: double (nullable = true)\n",
      " |-- _c528: double (nullable = true)\n",
      " |-- _c529: double (nullable = true)\n",
      " |-- _c530: double (nullable = true)\n",
      " |-- _c531: double (nullable = true)\n",
      " |-- _c532: double (nullable = true)\n",
      " |-- _c533: double (nullable = true)\n",
      " |-- _c534: double (nullable = true)\n",
      " |-- _c535: double (nullable = true)\n",
      " |-- _c536: double (nullable = true)\n",
      " |-- _c537: double (nullable = true)\n",
      " |-- _c538: double (nullable = true)\n",
      " |-- _c539: double (nullable = true)\n",
      " |-- _c540: double (nullable = true)\n",
      " |-- _c541: double (nullable = true)\n",
      " |-- _c542: double (nullable = true)\n",
      " |-- _c543: double (nullable = true)\n",
      " |-- _c544: double (nullable = true)\n",
      " |-- _c545: double (nullable = true)\n",
      " |-- _c546: double (nullable = true)\n",
      " |-- _c547: double (nullable = true)\n",
      " |-- _c548: double (nullable = true)\n",
      " |-- _c549: double (nullable = true)\n",
      " |-- _c550: double (nullable = true)\n",
      " |-- _c551: double (nullable = true)\n",
      " |-- _c552: double (nullable = true)\n",
      " |-- _c553: double (nullable = true)\n",
      " |-- _c554: double (nullable = true)\n",
      " |-- _c555: double (nullable = true)\n",
      " |-- _c556: double (nullable = true)\n",
      " |-- _c557: double (nullable = true)\n",
      " |-- _c558: double (nullable = true)\n",
      " |-- _c559: double (nullable = true)\n",
      " |-- _c560: double (nullable = true)\n",
      " |-- _c561: double (nullable = true)\n",
      " |-- _c562: double (nullable = true)\n",
      " |-- _c563: double (nullable = true)\n",
      " |-- _c564: double (nullable = true)\n",
      " |-- _c565: double (nullable = true)\n",
      " |-- _c566: double (nullable = true)\n",
      " |-- _c567: double (nullable = true)\n",
      " |-- _c568: double (nullable = true)\n",
      " |-- _c569: double (nullable = true)\n",
      " |-- _c570: double (nullable = true)\n",
      " |-- _c571: double (nullable = true)\n",
      " |-- _c572: double (nullable = true)\n",
      " |-- _c573: double (nullable = true)\n",
      " |-- _c574: double (nullable = true)\n",
      " |-- _c575: double (nullable = true)\n",
      " |-- _c576: double (nullable = true)\n",
      " |-- _c577: double (nullable = true)\n",
      " |-- _c578: double (nullable = true)\n",
      " |-- _c579: double (nullable = true)\n",
      " |-- _c580: double (nullable = true)\n",
      " |-- _c581: double (nullable = true)\n",
      " |-- _c582: double (nullable = true)\n",
      " |-- _c583: double (nullable = true)\n",
      " |-- _c584: double (nullable = true)\n",
      " |-- _c585: double (nullable = true)\n",
      " |-- _c586: double (nullable = true)\n",
      " |-- _c587: double (nullable = true)\n",
      " |-- _c588: double (nullable = true)\n",
      " |-- _c589: double (nullable = true)\n",
      " |-- _c590: double (nullable = true)\n",
      " |-- _c591: double (nullable = true)\n",
      " |-- _c592: double (nullable = true)\n",
      " |-- _c593: double (nullable = true)\n",
      " |-- _c594: double (nullable = true)\n",
      " |-- _c595: double (nullable = true)\n",
      " |-- _c596: double (nullable = true)\n",
      " |-- _c597: double (nullable = true)\n",
      " |-- _c598: double (nullable = true)\n",
      " |-- _c599: double (nullable = true)\n",
      " |-- _c600: double (nullable = true)\n",
      " |-- _c601: double (nullable = true)\n",
      " |-- _c602: double (nullable = true)\n",
      " |-- _c603: double (nullable = true)\n",
      " |-- _c604: double (nullable = true)\n",
      " |-- _c605: double (nullable = true)\n",
      " |-- _c606: double (nullable = true)\n",
      " |-- _c607: double (nullable = true)\n",
      " |-- _c608: double (nullable = true)\n",
      " |-- _c609: double (nullable = true)\n",
      " |-- _c610: double (nullable = true)\n",
      " |-- _c611: double (nullable = true)\n",
      " |-- _c612: double (nullable = true)\n",
      " |-- _c613: double (nullable = true)\n",
      " |-- _c614: double (nullable = true)\n",
      " |-- _c615: double (nullable = true)\n",
      " |-- _c616: double (nullable = true)\n",
      " |-- _c617: double (nullable = true)\n",
      " |-- _c618: double (nullable = true)\n",
      " |-- _c619: double (nullable = true)\n",
      " |-- _c620: double (nullable = true)\n",
      " |-- _c621: double (nullable = true)\n",
      " |-- _c622: double (nullable = true)\n",
      " |-- _c623: double (nullable = true)\n",
      " |-- _c624: double (nullable = true)\n",
      " |-- _c625: double (nullable = true)\n",
      " |-- _c626: double (nullable = true)\n",
      " |-- _c627: double (nullable = true)\n",
      " |-- _c628: double (nullable = true)\n",
      " |-- _c629: double (nullable = true)\n",
      " |-- _c630: double (nullable = true)\n",
      " |-- _c631: double (nullable = true)\n",
      " |-- _c632: double (nullable = true)\n",
      " |-- _c633: double (nullable = true)\n",
      " |-- _c634: double (nullable = true)\n",
      " |-- _c635: double (nullable = true)\n",
      " |-- _c636: double (nullable = true)\n",
      " |-- _c637: double (nullable = true)\n",
      " |-- _c638: double (nullable = true)\n",
      " |-- _c639: double (nullable = true)\n",
      " |-- _c640: double (nullable = true)\n",
      " |-- _c641: double (nullable = true)\n",
      " |-- _c642: double (nullable = true)\n",
      " |-- _c643: double (nullable = true)\n",
      " |-- _c644: double (nullable = true)\n",
      " |-- _c645: double (nullable = true)\n",
      " |-- _c646: double (nullable = true)\n",
      " |-- _c647: double (nullable = true)\n",
      " |-- _c648: double (nullable = true)\n",
      " |-- _c649: double (nullable = true)\n",
      " |-- _c650: double (nullable = true)\n",
      " |-- _c651: double (nullable = true)\n",
      " |-- _c652: double (nullable = true)\n",
      " |-- _c653: double (nullable = true)\n",
      " |-- _c654: double (nullable = true)\n",
      " |-- _c655: double (nullable = true)\n",
      " |-- _c656: double (nullable = true)\n",
      " |-- _c657: double (nullable = true)\n",
      " |-- _c658: double (nullable = true)\n",
      " |-- _c659: double (nullable = true)\n",
      " |-- _c660: double (nullable = true)\n",
      " |-- _c661: double (nullable = true)\n",
      " |-- _c662: double (nullable = true)\n",
      " |-- _c663: double (nullable = true)\n",
      " |-- _c664: double (nullable = true)\n",
      " |-- _c665: double (nullable = true)\n",
      " |-- _c666: double (nullable = true)\n",
      " |-- _c667: double (nullable = true)\n",
      " |-- _c668: double (nullable = true)\n",
      " |-- _c669: double (nullable = true)\n",
      " |-- _c670: double (nullable = true)\n",
      " |-- _c671: double (nullable = true)\n",
      " |-- _c672: double (nullable = true)\n",
      " |-- _c673: double (nullable = true)\n",
      " |-- _c674: double (nullable = true)\n",
      " |-- _c675: double (nullable = true)\n",
      " |-- _c676: double (nullable = true)\n",
      " |-- _c677: double (nullable = true)\n",
      " |-- _c678: double (nullable = true)\n",
      " |-- _c679: double (nullable = true)\n",
      " |-- _c680: double (nullable = true)\n",
      " |-- _c681: double (nullable = true)\n",
      " |-- _c682: double (nullable = true)\n",
      " |-- _c683: double (nullable = true)\n",
      " |-- _c684: double (nullable = true)\n",
      " |-- _c685: double (nullable = true)\n",
      " |-- _c686: double (nullable = true)\n",
      " |-- _c687: double (nullable = true)\n",
      " |-- _c688: double (nullable = true)\n",
      " |-- _c689: double (nullable = true)\n",
      " |-- _c690: double (nullable = true)\n",
      " |-- _c691: double (nullable = true)\n",
      " |-- _c692: double (nullable = true)\n",
      " |-- _c693: double (nullable = true)\n",
      " |-- _c694: double (nullable = true)\n",
      " |-- _c695: double (nullable = true)\n",
      " |-- _c696: double (nullable = true)\n",
      " |-- _c697: double (nullable = true)\n",
      " |-- _c698: double (nullable = true)\n",
      " |-- _c699: double (nullable = true)\n",
      " |-- _c700: double (nullable = true)\n",
      " |-- _c701: double (nullable = true)\n",
      " |-- _c702: double (nullable = true)\n",
      " |-- _c703: double (nullable = true)\n",
      " |-- _c704: double (nullable = true)\n",
      " |-- _c705: double (nullable = true)\n",
      " |-- _c706: double (nullable = true)\n",
      " |-- _c707: double (nullable = true)\n",
      " |-- _c708: double (nullable = true)\n",
      " |-- _c709: double (nullable = true)\n",
      " |-- _c710: double (nullable = true)\n",
      " |-- _c711: double (nullable = true)\n",
      " |-- _c712: double (nullable = true)\n",
      " |-- _c713: double (nullable = true)\n",
      " |-- _c714: double (nullable = true)\n",
      " |-- _c715: double (nullable = true)\n",
      " |-- _c716: double (nullable = true)\n",
      " |-- _c717: double (nullable = true)\n",
      " |-- _c718: double (nullable = true)\n",
      " |-- _c719: double (nullable = true)\n",
      " |-- _c720: double (nullable = true)\n",
      " |-- _c721: double (nullable = true)\n",
      " |-- _c722: double (nullable = true)\n",
      " |-- _c723: double (nullable = true)\n",
      " |-- _c724: double (nullable = true)\n",
      " |-- _c725: double (nullable = true)\n",
      " |-- _c726: double (nullable = true)\n",
      " |-- _c727: double (nullable = true)\n",
      " |-- _c728: double (nullable = true)\n",
      " |-- _c729: double (nullable = true)\n",
      " |-- _c730: double (nullable = true)\n",
      " |-- _c731: double (nullable = true)\n",
      " |-- _c732: double (nullable = true)\n",
      " |-- _c733: double (nullable = true)\n",
      " |-- _c734: double (nullable = true)\n",
      " |-- _c735: double (nullable = true)\n",
      " |-- _c736: double (nullable = true)\n",
      " |-- _c737: double (nullable = true)\n",
      " |-- _c738: double (nullable = true)\n",
      " |-- _c739: double (nullable = true)\n",
      " |-- _c740: double (nullable = true)\n",
      " |-- _c741: double (nullable = true)\n",
      " |-- _c742: double (nullable = true)\n",
      " |-- _c743: double (nullable = true)\n",
      " |-- _c744: double (nullable = true)\n",
      " |-- _c745: double (nullable = true)\n",
      " |-- _c746: double (nullable = true)\n",
      " |-- _c747: double (nullable = true)\n",
      " |-- _c748: double (nullable = true)\n",
      " |-- _c749: double (nullable = true)\n",
      " |-- _c750: double (nullable = true)\n",
      " |-- _c751: double (nullable = true)\n",
      " |-- _c752: double (nullable = true)\n",
      " |-- _c753: double (nullable = true)\n",
      " |-- _c754: double (nullable = true)\n",
      " |-- _c755: double (nullable = true)\n",
      " |-- _c756: double (nullable = true)\n",
      " |-- _c757: double (nullable = true)\n",
      " |-- _c758: double (nullable = true)\n",
      " |-- _c759: double (nullable = true)\n",
      " |-- _c760: double (nullable = true)\n",
      " |-- _c761: double (nullable = true)\n",
      " |-- _c762: double (nullable = true)\n",
      " |-- _c763: double (nullable = true)\n",
      " |-- _c764: double (nullable = true)\n",
      " |-- _c765: double (nullable = true)\n",
      " |-- _c766: double (nullable = true)\n",
      " |-- _c767: double (nullable = true)\n",
      " |-- _c768: double (nullable = true)\n",
      " |-- _c769: double (nullable = true)\n",
      " |-- _c770: double (nullable = true)\n",
      " |-- _c771: double (nullable = true)\n",
      " |-- _c772: double (nullable = true)\n",
      " |-- _c773: double (nullable = true)\n",
      " |-- _c774: double (nullable = true)\n",
      " |-- _c775: double (nullable = true)\n",
      " |-- _c776: double (nullable = true)\n",
      " |-- _c777: double (nullable = true)\n",
      " |-- _c778: double (nullable = true)\n",
      " |-- _c779: double (nullable = true)\n",
      " |-- _c780: double (nullable = true)\n",
      " |-- _c781: double (nullable = true)\n",
      " |-- _c782: double (nullable = true)\n",
      " |-- _c783: double (nullable = true)\n",
      " |-- _c784: double (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#print Schema\n",
    "df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Define Neural Network\n",
    "network = nn.Sequential(\n",
    "    nn.Linear(784, 256),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(256, 256),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(256, 10),\n",
    "    nn.Softmax(dim=1)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build the pytorch object\n",
    "torch_obj = serialize_torch_obj(\n",
    "    model=network,\n",
    "    criterion=nn.CrossEntropyLoss(),\n",
    "    optimizer=torch.optim.Adam,\n",
    "    lr=0.0001\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setup features\n",
    "vector_assembler = VectorAssembler(inputCols=df.columns[1:785], outputCol='features')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a SparkTorch Model with torch distributed. Barrier execution is on by default for this mode.\n",
    "spark_model = SparkTorch(\n",
    "    inputCol='features',\n",
    "    labelCol='_c0',\n",
    "    predictionCol='predictions',\n",
    "    torchObj=torch_obj,\n",
    "    iters=1000,\n",
    "    verbose=1,\n",
    "    miniBatch=256,\n",
    "    earlyStopPatience=40,\n",
    "    validationPct=0.2\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Can be used in a pipeline and saved.\n",
    "p = Pipeline(stages=[vector_assembler, spark_model]).fit(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#save\n",
    "#p.save('simple_dnn')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train accuracy = 0.787133\n"
     ]
    }
   ],
   "source": [
    "# Run predictions and evaluation\n",
    "predictions = p.transform(df).persist()\n",
    "evaluator = MulticlassClassificationEvaluator(labelCol=\"_c0\", predictionCol=\"predictions\", metricName=\"accuracy\")\n",
    "accuracy = evaluator.evaluate(predictions)\n",
    "print(\"Train accuracy = %g\" % accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#END"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}