spark_notebook_boilerplate.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "328ab9c5-d337-4115-9d82-57a16dfbea97",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from datetime import date, datetime\n",
    "\n",
    "import pandas as pd\n",
    "import pyspark\n",
    "import pyspark.sql\n",
    "import pyspark.sql.dataframe\n",
    "from pyspark.conf import *\n",
    "from pyspark.context import *\n",
    "from pyspark.sql import *\n",
    "from pyspark.sql import functions as F\n",
    "\n",
    "# from pyspark.sql.avro.functions import *\n",
    "from pyspark.sql.catalog import *\n",
    "from pyspark.sql.functions import *\n",
    "from pyspark.sql.types import *\n",
    "from pyspark.storagelevel import *\n",
    "\n",
    "os.environ[\"PYSPARK_PYTHON\"] = \"C:\\\\Users\\\\jmb\\\\.conda\\\\envs\\\\snake37Spark321\\\\python\"\n",
    "os.environ[\"PYSPARK_DRIVER_PYTHON\"] = \"C:\\\\Users\\\\jmb\\\\.conda\\\\envs\\\\snake37Spark321\\\\python\"\n",
    "\n",
    "SUBMIT_ARGS = \"--packages org.apache.spark:spark-avro_2.12:3.2.1 pyspark-shell\"\n",
    "os.environ[\"PYSPARK_SUBMIT_ARGS\"] = SUBMIT_ARGS\n",
    "\n",
    "\n",
    "spark = (\n",
    "    SparkSession.builder.master(\"local\")\n",
    "    .appName(\"Search\")\n",
    "    .config(\n",
    "        \"spark.jars.package\",\n",
    "        \"org.apache.spark:spark-avro_2.12:3.2.1,org.apache.spark:spark-sql-kafka-0-10_2.12.12:3.2.1\",\n",
    "    )\n",
    "    .config(\"spark.serializer\", \"org.apache.spark.serializer.KryoSerializer\")\n",
    "    .config(conf=SparkConf())\n",
    "    .getOrCreate()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f3f68b5a-9854-464e-9b15-29f128af4f0d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'http://DESKTOP-OVJ9BOU:4040'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark.sparkContext.getConf().getAll()\n",
    "spark._jsc.sc().uiWebUrl().get()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "adc97a5f-46d2-4143-9024-ea96b8d2ec86",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f339b7e3-6fc3-4026-8b75-d483914fc83d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+---+-------+----------+-------------------+----------------------------------+\n",
      "|a  |b  |c      |d         |e                  |meta_oracle_cdc_precisiontimestamp|\n",
      "+---+---+-------+----------+-------------------+----------------------------------+\n",
      "|1  |2.0|string1|2000-01-01|2000-01-01 12:00:00|2022-04-12 19:15:34.0             |\n",
      "|2  |3.0|string2|2000-02-01|2000-01-02 12:00:00|2022-04-12 19:15:34.0             |\n",
      "|3  |4.0|string3|2000-03-01|2000-01-03 12:00:00|2022-04-12 19:15:34.0             |\n",
      "+---+---+-------+----------+-------------------+----------------------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df = spark.createDataFrame(\n",
    "    [\n",
    "        (\n",
    "            1,\n",
    "            2.0,\n",
    "            \"string1\",\n",
    "            date(2000, 1, 1),\n",
    "            datetime(2000, 1, 1, 12, 0),\n",
    "            \"2022-04-12 19:15:34.0\",\n",
    "        ),\n",
    "        (\n",
    "            2,\n",
    "            3.0,\n",
    "            \"string2\",\n",
    "            date(2000, 2, 1),\n",
    "            datetime(2000, 1, 2, 12, 0),\n",
    "            \"2022-04-12 19:15:34.0\",\n",
    "        ),\n",
    "        (\n",
    "            3,\n",
    "            4.0,\n",
    "            \"string3\",\n",
    "            date(2000, 3, 1),\n",
    "            datetime(2000, 1, 3, 12, 0),\n",
    "            \"2022-04-12 19:15:34.0\",\n",
    "        ),\n",
    "    ],\n",
    "    schema=\"a long, b double, c string, d date, e timestamp, meta_oracle_cdc_precisiontimestamp string\",\n",
    ")\n",
    "df_t0 = df.sort(*df.columns)\n",
    "_ = df_t0.createOrReplaceTempView(\"cc_activity_vw\")\n",
    "df_tt = spark.sql(\"select  *  from cc_activity_vw\")\n",
    "df_tt.show(truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d46bfced-1368-402e-8d8f-7a16dae0f3ff",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- a: long (nullable = true)\n",
      " |-- b: double (nullable = true)\n",
      " |-- c: string (nullable = true)\n",
      " |-- d: date (nullable = true)\n",
      " |-- e: timestamp (nullable = true)\n",
      " |-- meta_oracle_cdc_precisiontimestamp: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_tt.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a07ef4ce-9d53-4c00-a19c-4c651945958f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}