PySpark_upload_data_to_DB_Script.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "98ba0ffb-e9a3-4103-ac86-38f73a8c5245",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "# Read data from a CSV file into a DataFrame\n",
    "\n",
    "df1 = spark.read.option(\"header\", True).option(\"inferschema\", True).csv(\"/mnt/Deltalake/Bronze/Ride_Rating_Table.csv\")\n",
    "\n",
    "df2 = spark.read.option(\"header\", True).option(\"inferschema\", True).csv(\"/mnt/Deltalake/Bronze/Trip_Transaction_Table.csv\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "35e1b3e8-4c4f-4468-8849-10143df7457b",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "# Define the server name for the SQL Server\n",
    "server_name = \"jdbc:sqlserver://rgadf.database.windows.net\"\n",
    "\n",
    "# Define the name of the SQL Server database\n",
    "database_name = \"rgadf\"\n",
    "\n",
    "# Define the connection URL for JDBC with authentication details and other connection parameters\n",
    "url = 'jdbc:sqlserver://rgadf.database.windows.net:1433;database=rgadf;user=rgadf@rgadf;password=<>;encrypt=true;trustServerCertificate=false;hostNameInCertificate=*.database.windows.net;loginTimeout=30';\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "96183ac1-7c9c-4936-b711-227c50e7fcbf",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "# Define the name of the SQL Server table\n",
    "table_name = \"dbo.ride_ratings1\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "39cb2c32-db24-44b8-af78-1cc13bb99590",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "# Write the DataFrame df1 to a SQL Server table using the JDBC write method\n",
    "df1.write.jdbc(url, table=table_name, mode=\"append\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "1e25c182-4c58-432f-a806-96cd2d5e7878",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "# Write the DataFrame df2 to a SQL Server table using the JDBC write method\n",
    "df2.write.jdbc(url, table='dbo.trip_transactions_1', mode='append')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "application/vnd.databricks.v1+notebook": {
   "dashboards": [],
   "language": "python",
   "notebookMetadata": {
    "mostRecentlyExecutedCommandWithImplicitDF": {
     "commandId": 1812374475256350,
     "dataframes": [
      "_sqldf"
     ]
    },
    "pythonIndentUnit": 4
   },
   "notebookName": "PySpark_DB_Script",
   "notebookOrigID": 1812374475256214,
   "widgets": {}
  },
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}