Presented on final lesson

Signed-off-by: cnt7-naya-cdh6 <naya@cnt7-naya-cdh6.org>

Author: cnt7-naya-cdh6

Dashboard.ipynb

@@ -25,14 +25,15 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "hdfs_host = 'localhost'\n",
-    "hdfs_port = 9870\n",
-    "hive_port = 10000\n",
-    "hive_username = 'hdfs'\n",
-    "hive_password = 'naya'\n",
-    "hive_database = 'twitter'\n",
-    "hive_mode = 'CUSTOM'\n",
-    "\n",
+    "from common_vars import *"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
     "hive_cnx = hive.Connection(\n",
     "  host = hdfs_host, \n",
     "  port = hive_port, \n",
@@ -156,7 +157,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "q = .95\n",
+    "q = .7\n",
     "qS = abs(t.Sentiment).quantile(q)\n",
     "qS"
    ]
@@ -169,7 +170,7 @@
    "source": [
     "plt.figure(figsize=[13,9])\n",
     "plt.hist(t.Sentiment[t.Sentiment!=0], bins=200)\n",
-    "plt.vlines([-qS,qS],ymin= 0, ymax = 600,colors='red')"
+    "plt.vlines([-qS,qS],ymin= 0, ymax = 10,colors='red')"
    ]
   },
   {
@@ -190,7 +191,7 @@
    "source": [
     "plt.figure(figsize=[9,9])\n",
     "plt.hist(t.user_followers[t.Sentiment!=0], bins=200)\n",
-    "plt.vlines([quf],ymin= 0, ymax = 350,colors='red')"
+    "plt.vlines([quf],ymin= 0, ymax = 5,colors='red')"
    ]
   },
   {
@@ -199,7 +200,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "tt = (t.user_followers>quf) & (t.user_followers<5) & ((t.Sentiment) < -qS)\n"
+    "#tt = (t.user_followers>quf) & (t.user_followers<5) & ((t.Sentiment) < -qS)\n",
+    "tt =  ((t.Sentiment) < -qS)\n"
    ]
   },
   {
@@ -221,7 +223,7 @@
     "badboys = tt.sort_values(by=['Sentiment','user_followers'], ascending = [True, False] )\n",
     "cols = \"text created_at user_followers Sentiment\".split(\" \")\n",
     "\n",
-    "print(\"\\n-------\\n\".join(badboys['text']))"
+    "print(\"\\n-----------------------------\\n\".join(badboys['text']))"
    ]
   },
   {
@@ -254,8 +256,8 @@
    },
    "outputs": [],
    "source": [
-    "print(badboys.text[i])\n",
-    "i+=1"
+    "# print(badboys.text[i])\n",
+    "# i+=1"
    ]
   },
   {
@@ -264,8 +266,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "a,b = min(t['created_at']), max(t['created_at'])\n",
-    "a,b"
+    "# a,b = min(t['created_at']), max(t['created_at'])\n",
+    "# a,b"
    ]
   },
   {

HDFS_DB_Init.ipynb

@@ -332,112 +332,6 @@
    "source": [
     "run_queries(q1)"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from pyspark.sql.types import StructType, StringType, MapType, StructField,\\\n",
-    "                              BooleanType, DateType, NumericType, IntegerType,\\\n",
-    "                              LongType, TimestampType, FloatType, ArrayType\n",
-    "import pyarrow as pa\n",
-    "\n",
-    "\n",
-    "topic = 'TweeterArchive'\n",
-    "partitionCol = \"created_ym\"\n",
-    "Keywords = 'Israel'\n",
-    "\n",
-    "\n",
-    "hdfs_host = 'localhost'\n",
-    "hdfs_port = 9870\n",
-    "hive_port = 10000\n",
-    "hive_username = 'hdfs'\n",
-    "hive_password = 'naya'\n",
-    "hive_database = 'twitter'\n",
-    "hive_mode = 'CUSTOM'\n",
-    "\n",
-    "\n",
-    "fs = pa.hdfs.connect(\n",
-    "    host=hdfs_host, \n",
-    "    port=8020, \n",
-    "    user=hive_username, \n",
-    "    kerb_ticket=None, \n",
-    "    driver='libhdfs', \n",
-    "    extra_conf=None)\n",
-    "\n",
-    "\n",
-    "event_fields = [ 'id', 'text','created_at', 'geo', 'coordinates', 'place',\n",
-    "                 'quote_count', 'reply_count', 'retweet_count', 'favorite_count' ]\n",
-    "\n",
-    "tweet_keys =  event_fields + ['user_id', 'user_followers' ]\n",
-    "\n",
-    "tweet_types = [LongType, StringType, TimestampType, StringType, StringType, StringType, \n",
-    "               IntegerType, IntegerType, IntegerType, IntegerType, LongType, IntegerType]\n",
-    "\n",
-    "# tweet_types  = [StringType]* 11 \n",
-    "\n",
-    "\n",
-    "user_fields = ['id', 'name', 'screen_name','created_at', 'location', 'url',\n",
-    "                         'protected', 'verified', 'followers_count', 'friends_count',\n",
-    "                         'listed_count', 'favourites_count', 'statuses_count', 'withheld_in_countries']\n",
-    "\n",
-    "user_keys = user_fields + [partitionCol]\n",
-    "\n",
-    "user_types = [LongType, StringType, StringType, TimestampType, StringType, StringType, \n",
-    "              BooleanType, BooleanType, IntegerType, IntegerType, \n",
-    "              IntegerType, IntegerType, IntegerType, StringType, StringType]\n",
-    "\n",
-    "# user_types = [StringType]* 14\n",
-    "\n",
-    "\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "conf.py"
-   ]
   }
  ],
  "metadata": {