From 3292b3fdaf055f7323b97b4b28e658726ab05526 Mon Sep 17 00:00:00 2001 From: Nitish Ramakrishnan <69108657+niccolopaganini@users.noreply.github.com> Date: Wed, 13 Sep 2023 15:14:37 -0600 Subject: [PATCH] Delete bin/API_Migration_Automation directory 1. Deleting this as it is not necessary 2. Will replace it with a new directory --- .../Changes for script.ipynb | 430 ------------------ bin/API_Migration_Automation/environment.yml | 13 - 2 files changed, 443 deletions(-) delete mode 100644 bin/API_Migration_Automation/Changes for script.ipynb delete mode 100644 bin/API_Migration_Automation/environment.yml diff --git a/bin/API_Migration_Automation/Changes for script.ipynb b/bin/API_Migration_Automation/Changes for script.ipynb deleted file mode 100644 index 0333aa43c..000000000 --- a/bin/API_Migration_Automation/Changes for script.ipynb +++ /dev/null @@ -1,430 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "46be159b", - "metadata": {}, - "outputs": [], - "source": [ - "#imports\n", - "import os\n", - "import csv\n", - "import requests\n", - "import bs4\n", - "import subprocess\n", - "import re" - ] - }, - { - "cell_type": "markdown", - "id": "338e32fc", - "metadata": {}, - "source": [ - "### URL -> need to be changed to take an input from the user" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d73fe4d9", - "metadata": {}, - "outputs": [], - "source": [ - "url = \"https://developer.android.com/sdk/api_diff/33/changes/alldiffs_index_changes\" #URL from android website - only changes\n", - "\n", - "#Request and creation of Beautiful Soup with the response\n", - "response = requests.get(url)\n", - "soup = bs4.BeautifulSoup(response.content, \"html.parser\")\n", - "\n", - "#getting links from the webpage\n", - "links = []\n", - "for a in soup.find_all(\"a\"):\n", - " if a.has_attr(\"href\") and a[\"href\"].startswith(\"/sdk/api_diff/33/changes/\"):\n", - " links.append(a[\"href\"])\n", - "\n", - "#creation and writing the file with DictWriter\n", - "csv_file = open(\"classes.csv\", \"w\", newline=\"\")\n", - "csv_writer = csv.DictWriter(csv_file, fieldnames=[\"Package Class\"])\n", - "csv_writer.writeheader()\n", - "\n", - "#technically not writing any links but the classes themselves\n", - "for link in links:\n", - " css_class = link.split('/')[-1]\n", - " csv_writer.writerow({\"Package Class\": css_class})\n", - "\n", - "\n", - "#JSON not required\n", - "#json_file = open(\"classes.json\", \"w\")\n", - "#json_data = {\"links\": links}\n", - "#json.dump(json_data, json_file)" - ] - }, - { - "cell_type": "markdown", - "id": "2456a98b", - "metadata": {}, - "source": [ - "### Takes the classes.csv file and strips unnecessary info" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "44d4b6c8", - "metadata": {}, - "outputs": [], - "source": [ - "def modify_csv(csv_path):\n", - " with open(csv_path, \"r\") as f:\n", - " reader = csv.DictReader(f)\n", - " with open(\"modified_csv.csv\", \"w\") as fw:\n", - " writer = csv.DictWriter(fw, fieldnames=[\"Package Class\"])\n", - " writer.writeheader()\n", - " for row in reader:\n", - " class_name = row[\"Package Class\"]\n", - " class_name = class_name.split(\"#\")[0]\n", - " class_name = class_name.lstrip(\".\")\n", - " class_name = class_name.strip(\"[]\")\n", - " class_name = class_name.rstrip(\".\")\n", - " row[\"Package Class\"] = class_name\n", - " writer.writerow(row)\n", - "\n", - "if __name__ == \"__main__\":\n", - " csv_path = \"classes.csv\"\n", - " modify_csv(csv_path)" - ] - }, - { - "cell_type": "markdown", - "id": "9361de0a", - "metadata": {}, - "source": [ - "### takes the CSV and compiles it into a unique list" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "73f28fe8", - "metadata": {}, - "outputs": [], - "source": [ - "with open(\"modified_csv.csv\", \"r\") as f:\n", - " classes = set()\n", - " for line in f:\n", - " class_name = line.strip().split(\",\")[0]\n", - " classes.add(class_name)\n", - "\n", - "with open(\"unique_classes.csv\", \"w\") as fw:\n", - " writer = csv.writer(fw)\n", - " for class_name in classes:\n", - " writer.writerow([class_name])" - ] - }, - { - "cell_type": "markdown", - "id": "5616eee1", - "metadata": {}, - "source": [ - "### Only retain the last part" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b75da672", - "metadata": {}, - "outputs": [], - "source": [ - "def simplify(csv_file, output_file):\n", - " with open(csv_file, \"r\") as f:\n", - " reader = csv.reader(f)\n", - " lines = []\n", - " for row in reader:\n", - " new_row = []\n", - " for item in row:\n", - " end_index = item.rfind(\".\")\n", - " if end_index == -1:\n", - " new_row.append(item)\n", - " else:\n", - " simplified_item = item[end_index + 1:]\n", - " new_row.append(simplified_item)\n", - " lines.append(new_row)\n", - "\n", - " with open(output_file, \"w\") as f:\n", - " writer = csv.writer(f)\n", - " writer.writerows(lines)\n", - "\n", - "\n", - "if __name__ == \"__main__\":\n", - " csv_file = \"unique_classes.csv\"\n", - " output_file = \"simplify_classes.csv\"\n", - " simplify(csv_file, output_file)\n" - ] - }, - { - "cell_type": "markdown", - "id": "3ba64e82", - "metadata": {}, - "source": [ - "### finds matching classes and appends them into a CSV" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ee57000e", - "metadata": {}, - "outputs": [], - "source": [ - "def match_csv_java(directory, csv_file):\n", - " java_files = []\n", - " for root, directories, files in os.walk(directory):\n", - " for file in files:\n", - " if file.endswith(\".java\"):\n", - " java_files.append(os.path.join(root, file))\n", - "\n", - " if os.path.exists(csv_file):\n", - " with open(os.path.join(directory, csv_file), \"r\") as f:\n", - " reader = csv.reader(f)\n", - " for row in reader:\n", - " if row:\n", - " classes = row[0]\n", - "\n", - " for java_file in java_files:\n", - " with open(java_file, \"r\", encoding=\"utf-8\") as f:\n", - " text = f.read()\n", - " if classes in text:\n", - " with open(\"output.csv\", \"a\") as csvfile:\n", - " csvwriter = csv.writer(csvfile)\n", - " csvwriter.writerow([classes, java_file])\n", - "\n", - "\n", - "if __name__ == \"__main__\":\n", - " match_csv_java(\"/Users/nvsr/Downloads/e-mission-phone-nvsr-iter1-API_Migration_aid_script_patch-1/plugins\", \"/Users/nvsr/Downloads/e-mission-phone-nvsr-iter1-API_Migration_aid_script_patch-1/bin/API_Migration_scripts_readme/simplify_classes.csv\")\n" - ] - }, - { - "cell_type": "markdown", - "id": "c060420c", - "metadata": {}, - "source": [ - "### Removes the 2nd column which has the PATH and creates a unique" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c51bc20f", - "metadata": {}, - "outputs": [], - "source": [ - "def get_unique_packages(csv_file):\n", - " with open(csv_file, \"r\") as f:\n", - " reader = csv.reader(f)\n", - " packages = set()\n", - " for row in reader:\n", - " if row:\n", - " packages.add(row[0])\n", - "\n", - " with open(\"unique_packages.csv\", \"w\") as f:\n", - " csvwriter = csv.writer(f)\n", - " for package in packages:\n", - " csvwriter.writerow([package])\n", - "\n", - "if __name__ == \"__main__\":\n", - " get_unique_packages(\"output.csv\")" - ] - }, - { - "cell_type": "markdown", - "id": "d6bc3c5c", - "metadata": {}, - "source": [ - "# Now the process is repeated by scraping but this time, we save them as links and then use those to scrape" - ] - }, - { - "cell_type": "markdown", - "id": "48bf4cd3", - "metadata": {}, - "source": [ - "### Scrape links" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "53cafc4a", - "metadata": {}, - "outputs": [], - "source": [ - "def get_links(url):\n", - " response = requests.get(url)\n", - "\n", - " if response.status_code == 200:\n", - " soup = bs4.BeautifulSoup(response.content, \"html.parser\")\n", - "\n", - " links = []\n", - "\n", - " for a in soup.find_all(\"a\"):\n", - " if \"href\" in a.attrs:\n", - " links.append(a[\"href\"])\n", - "\n", - " return links\n", - "\n", - "def main():\n", - " links = get_links(\"https://developer.android.com/sdk/api_diff/33/changes/alldiffs_index_changes\")\n", - "\n", - " with open(\"links.csv\", \"w\", newline=\"\") as f:\n", - " csvwriter = csv.writer(f)\n", - " for link in links:\n", - " csvwriter.writerow([link])\n", - "\n", - "if __name__ == \"__main__\":\n", - " main()" - ] - }, - { - "cell_type": "markdown", - "id": "895648c6", - "metadata": {}, - "source": [ - "### take the above CSV and simplify them" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "79e6c3ac", - "metadata": {}, - "outputs": [], - "source": [ - "def filter_links(csv_file, output_file):\n", - " with open(csv_file, \"r\") as f:\n", - " reader = csv.reader(f)\n", - " links = []\n", - " for row in reader:\n", - " if row[0].startswith(\"/sdk\"):\n", - " links.append(row)\n", - "\n", - " with open(output_file, \"w\") as f:\n", - " csvwriter = csv.writer(f)\n", - " csvwriter.writerows(links)\n", - "\n", - "\n", - "if __name__ == \"__main__\":\n", - " filter_links(\"links.csv\", \"simplified_links.csv\")" - ] - }, - { - "cell_type": "markdown", - "id": "d2dca327", - "metadata": {}, - "source": [ - "### Gets the methods" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d9237191", - "metadata": {}, - "outputs": [], - "source": [ - "def get_contents(url):\n", - " response = requests.get(f\"https://developer.android.com{url}\")\n", - "\n", - " if response.status_code == 200:\n", - " soup = bs4.BeautifulSoup(response.content, \"html.parser\")\n", - "\n", - " tables = soup.find_all(\"table\")\n", - "\n", - " contents = []\n", - "\n", - " for table in tables:\n", - " table_contents = []\n", - " for row in table.find_all(\"tr\"):\n", - " row_contents = []\n", - " for cell in row.find_all(\"td\"):\n", - " row_contents.append(cell.text)\n", - " table_contents.append(row_contents)\n", - " contents.append(table_contents)\n", - "\n", - " return contents\n", - "\n", - "def main():\n", - " with open(\"simplified_links.csv\", \"r\") as f:\n", - " reader = csv.reader(f)\n", - " links = []\n", - " for row in reader:\n", - " links.append(row[0])\n", - "\n", - " new_contents = []\n", - "\n", - " for link in links:\n", - " contents = get_contents(link)\n", - "\n", - " with open(\"methods_scraped.csv\", \"a\") as f:\n", - " csvwriter = csv.writer(f)\n", - " csvwriter.writerow([link] + contents)\n", - "\n", - "if __name__ == \"__main__\":\n", - " main()\n" - ] - }, - { - "cell_type": "markdown", - "id": "6a636b3e", - "metadata": {}, - "source": [ - "### Removes unncessary elements" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "546b6b32", - "metadata": {}, - "outputs": [], - "source": [ - "def remove_columns(csv_file, output_file, column_numbers):\n", - " with open(csv_file, \"r\") as f:\n", - " reader = csv.reader(f)\n", - " rows = []\n", - " for row in reader:\n", - " new_row = [row[i] for i in range(len(row)) if i not in column_numbers]\n", - " rows.append(new_row)\n", - "\n", - " with open(output_file, \"w\") as f:\n", - " csvwriter = csv.writer(f)\n", - " csvwriter.writerows(rows)\n", - "\n", - "\n", - "remove_columns(\"methods_scraped.csv\", \"methods.csv\", [1, 2])" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/bin/API_Migration_Automation/environment.yml b/bin/API_Migration_Automation/environment.yml deleted file mode 100644 index 087901965..000000000 --- a/bin/API_Migration_Automation/environment.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: API_migration_scanner -channels: - - defaults -dependencies: - - _anaconda_depends - - python=3.11 - - os - - csv - - requests - - bs4 - - re - - subprocess -prefix: /Users/nseptank/anaconda3/envs/API_migration_scanner