01 Worldwide Earthquake Events API - Bronze Layer Processing.ipynb

{"cells":[{"cell_type":"markdown","source":["## Worldwide Earthquake Events API - Bronze Layer Processing"],"metadata":{"nteract":{"transient":{"deleting":false}}},"id":"d00c0b9d-b7ea-4188-acc9-0bdb5049f678"},{"cell_type":"code","source":["import requests\n","import json"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"statement_id":15,"statement_ids":[15],"state":"finished","livy_statement_state":"available","session_id":"87780d67-e6f1-4b5a-acb4-3b330d8f1226","normalized_state":"finished","queued_time":"2024-09-21T16:36:36.3795299Z","session_start_time":null,"execution_start_time":"2024-09-21T16:36:36.8416876Z","execution_finish_time":"2024-09-21T16:36:37.093608Z","parent_msg_id":"8d4ee76e-36e3-4671-9611-b1d4018d1a01"},"text/plain":"StatementMeta(, 87780d67-e6f1-4b5a-acb4-3b330d8f1226, 15, Finished, Available, Finished)"},"metadata":{}}],"execution_count":13,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"234df52a-7a21-4621-8c46-f65eb64244db"},{"cell_type":"markdown","source":["## Fetching and Saving the data as json"],"metadata":{"nteract":{"transient":{"deleting":false}}},"id":"0bfa06af-4527-4878-99e3-4fe983d5b47b"},{"cell_type":"code","source":["# Construct the API URL with start and end date provided by Data Factory, formatted for geojson output.\n","url = f\"https://earthquake.usgs.gov/fdsnws/event/1/query?format=geojson&starttime={start_date}&endtime={end_date}\"\n","\n","# Make the get request to fetch data\n","response = requests.get(url)\n","\n","# Checking if the request was successful\n","if response.status_code == 200:\n","    #Get the JSON response\n","    data = response.json()\n","    data = data['features']\n","\n","    # File name for data to be store in\n","    file_path = f'/lakehouse/default/Files/{start_date}_earthquake.json'\n","\n","    # Saving the data in the specified file\n","    with open(file_path, 'w') as file:\n","        json.dump(data, file, indent=4)\n","\n","    print(f\"Data Successfully saved to {file_path}\")\n","\n","else:\n","    print(f\"Failed to fetch data. Status Code: {response.status_code}\")"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"statement_id":19,"statement_ids":[19],"state":"finished","livy_statement_state":"available","session_id":"87780d67-e6f1-4b5a-acb4-3b330d8f1226","normalized_state":"finished","queued_time":"2024-09-21T16:40:26.1395532Z","session_start_time":null,"execution_start_time":"2024-09-21T16:40:28.1609789Z","execution_finish_time":"2024-09-21T16:40:31.3270701Z","parent_msg_id":"21ca6997-079f-4a49-8fb5-1e55e8ac7b83"},"text/plain":"StatementMeta(, 87780d67-e6f1-4b5a-acb4-3b330d8f1226, 19, Finished, Available, Finished)"},"metadata":{}},{"output_type":"stream","name":"stdout","text":["Data Successfully saved to /lakehouse/default/Files/2024-09-14_earthquake.json\n"]}],"execution_count":17,"metadata":{"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"0921a5d8-e6c2-4d9b-81c5-89c0371b4a16"}],"metadata":{"kernel_info":{"name":"synapse_pyspark"},"kernelspec":{"name":"synapse_pyspark","language":"Python","display_name":"Synapse PySpark"},"language_info":{"name":"python"},"microsoft":{"language":"python","language_group":"synapse_pyspark","ms_spell_check":{"ms_spell_check_language":"en"}},"widgets":{},"nteract":{"version":"nteract-front-end@1.0.0"},"spark_compute":{"compute_id":"/trident/default"},"dependencies":{"lakehouse":{"default_lakehouse":"9dc1b70c-c86d-4929-9933-585658bb9da9","known_lakehouses":[{"id":"9dc1b70c-c86d-4929-9933-585658bb9da9"}],"default_lakehouse_name":"Earthquake_Lakehouse","default_lakehouse_workspace_id":"99b06628-c37b-4fa7-819b-eebf5b8ad847"}}},"nbformat":4,"nbformat_minor":5}