Rep_entity_merger_and_extractor.ipynb.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c34d4f7",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-10-30T13:21:19.949775Z",
     "iopub.status.busy": "2023-10-30T13:21:19.949404Z",
     "iopub.status.idle": "2023-10-30T13:21:35.899067Z",
     "shell.execute_reply": "2023-10-30T13:21:35.897486Z"
    },
    "papermill": {
     "duration": 15.960036,
     "end_time": "2023-10-30T13:21:35.901981",
     "exception": false,
     "start_time": "2023-10-30T13:21:19.941945",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Installation for IOC defanging - https://github.com/ioc-fang/ioc-fanger\n",
    "!pip install -q ioc-fanger"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "245f572a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-30T13:21:35.916328Z",
     "iopub.status.busy": "2023-10-30T13:21:35.915890Z",
     "iopub.status.idle": "2023-10-30T13:21:36.412779Z",
     "shell.execute_reply": "2023-10-30T13:21:36.411453Z"
    },
    "papermill": {
     "duration": 0.506564,
     "end_time": "2023-10-30T13:21:36.415803",
     "exception": false,
     "start_time": "2023-10-30T13:21:35.909239",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import ioc_fanger\n",
    "import pandas as pd\n",
    "import re\n",
    "import json\n",
    "from tqdm import tqdm\n",
    "import ipaddress"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c5919a0",
   "metadata": {
    "papermill": {
     "duration": 0.005215,
     "end_time": "2023-10-30T13:21:36.426302",
     "exception": false,
     "start_time": "2023-10-30T13:21:36.421087",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Merger"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d892a38",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-30T13:21:36.439378Z",
     "iopub.status.busy": "2023-10-30T13:21:36.438819Z",
     "iopub.status.idle": "2023-10-30T13:22:00.882018Z",
     "shell.execute_reply": "2023-10-30T13:22:00.880757Z"
    },
    "papermill": {
     "duration": 24.452913,
     "end_time": "2023-10-30T13:22:00.884725",
     "exception": false,
     "start_time": "2023-10-30T13:21:36.431812",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "total_data = []\n",
    "for i in tqdm(range(1, 394), desc=\"Merging\"):\n",
    "    file_path = f'/home/anon/input/chunks-predictions/REP_chunks_predictions/REP_{i}.json'\n",
    "    with open(file_path, 'r', encoding='utf-8') as file:\n",
    "        data = json.load(file)\n",
    "    \n",
    "    for entry in data:\n",
    "        total_data.append(entry)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "df587464",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-30T13:22:00.925277Z",
     "iopub.status.busy": "2023-10-30T13:22:00.924873Z",
     "iopub.status.idle": "2023-10-30T13:22:00.933549Z",
     "shell.execute_reply": "2023-10-30T13:22:00.932424Z"
    },
    "papermill": {
     "duration": 0.031858,
     "end_time": "2023-10-30T13:22:00.935970",
     "exception": false,
     "start_time": "2023-10-30T13:22:00.904112",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "len(total_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab711771",
   "metadata": {
    "papermill": {
     "duration": 0.018713,
     "end_time": "2023-10-30T13:22:00.973790",
     "exception": false,
     "start_time": "2023-10-30T13:22:00.955077",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Extractor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "416d8ae0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-30T13:22:01.012546Z",
     "iopub.status.busy": "2023-10-30T13:22:01.012110Z",
     "iopub.status.idle": "2023-10-30T13:22:01.022809Z",
     "shell.execute_reply": "2023-10-30T13:22:01.021617Z"
    },
    "papermill": {
     "duration": 0.032975,
     "end_time": "2023-10-30T13:22:01.025393",
     "exception": false,
     "start_time": "2023-10-30T13:22:00.992418",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Regular expressions to match different patterns\n",
    "email_regex = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$'\n",
    "time_regex = r'^\\d{1,2}:\\d{1,2}(:\\d{1,2})?$'\n",
    "ip_regex = r'^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}$'\n",
    "web_domain_regex = r'^([a-zA-Z0-9-]+\\.[a-zA-Z]{2,})$'\n",
    "url_regex = r'^https?://[^\\s/$.?#][^\\s/?#]*[^\\s/.?#]$'\n",
    "file_with_extension_regex = r'^[a-zA-Z0-9-]+\\.[a-zA-Z0-9]+$'\n",
    "md5_regex = r'^[a-fA-F0-9]{32}$'\n",
    "sha1_regex = r'^[a-fA-F0-9]{40}$'\n",
    "sha2_regex = r'^[a-fA-F0-9]{64}$'\n",
    "malware_name_regex = r'^[a-zA-Z0-9-]+(/[a-zA-Z0-9-]+)+$'\n",
    "cve_regex = r'^CVE-[0-9]{4}-[0-9]{4,}$'\n",
    "vulnerability_id_regex = r'^[a-zA-Z0-9-]+$'\n",
    "windows_environment_variable_regex = r'^%(WINDOWS|[a-zA-Z0-9_]+)%.*?(\\\\)?$'\n",
    "unix_like_environment_variable_regex = r'^\\$\\{[a-zA-Z_][a-zA-Z0-9_]*\\}.*?(\\\\)?$'\n",
    "\n",
    "def clean_keyword(word):\n",
    "    # Check if the word matches any of the special formats\n",
    "    special_formats = [\n",
    "        email_regex, time_regex, ip_regex, web_domain_regex, url_regex,\n",
    "        file_with_extension_regex, md5_regex, sha1_regex, sha2_regex, malware_name_regex,\n",
    "        cve_regex, vulnerability_id_regex, windows_environment_variable_regex, unix_like_environment_variable_regex\n",
    "    ]\n",
    "    for regex_pattern in special_formats:\n",
    "        if re.match(regex_pattern, word):\n",
    "            return word\n",
    "\n",
    "    if len(word.split()) == 1:\n",
    "        # Remove punctuation at the beginning and ending of the word\n",
    "        cleaned_word = re.sub(r'^\\W+|(^\\W+)|\\W+$', r'\\1', word)\n",
    "    else:\n",
    "        cleaned_word = re.sub(r'^\\W+|\\W+$', '', word)\n",
    "\n",
    "    \"\"\"if word != cleaned_word:\n",
    "        print('\\t{0} --> {1}'.format(word, cleaned_word))\"\"\"\n",
    "\n",
    "    return cleaned_word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "488da257",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-30T13:22:01.066164Z",
     "iopub.status.busy": "2023-10-30T13:22:01.065772Z",
     "iopub.status.idle": "2023-10-30T13:22:01.084386Z",
     "shell.execute_reply": "2023-10-30T13:22:01.083514Z"
    },
    "papermill": {
     "duration": 0.042329,
     "end_time": "2023-10-30T13:22:01.086685",
     "exception": false,
     "start_time": "2023-10-30T13:22:01.044356",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def extract_entities(row):\n",
    "    entity_dict = {\n",
    "        \"ACT\": set(),\n",
    "        \"APT\": set(),\n",
    "        \"DOM\": set(),\n",
    "        \"EMAIL\": set(),\n",
    "        \"ENCR\": set(),\n",
    "        \"FILE\": set(),\n",
    "        \"IDTY\": set(),\n",
    "        \"IP\": set(),\n",
    "        \"LOC\": set(),\n",
    "        \"MAL\": set(),\n",
    "        \"MD5\": set(),\n",
    "        \"OS\": set(),\n",
    "        \"PROT\": set(),\n",
    "        \"SECTEAM\": set(),\n",
    "        \"SHA1\": set(),\n",
    "        \"SHA2\": set(),\n",
    "        \"TIME\": set(),\n",
    "        \"TOOL\": set(),\n",
    "        \"URL\": set(),\n",
    "        \"VULID\": set(),\n",
    "        \"VULNAME\": set()\n",
    "    }\n",
    "\n",
    "    if int(row['ID']) % 1000 == 0:\n",
    "        print('\\n-------------------------ROW {}\\\\75405 - {:.2%}\\n'\n",
    "              .format(row['ID'], int(row['ID'])/75405))\n",
    "\n",
    "    sentences_list = [string.split() for string in row['content']]\n",
    "    tags_list = [string.split(',') for string in row['tags']]\n",
    "\n",
    "    # Loop over sentences and tags simultaneously\n",
    "    \n",
    "    for sentence, sentence_tags in zip(sentences_list, tags_list):\n",
    "\n",
    "        if len(sentence) != len(sentence_tags):\n",
    "            sentence_tags = sentence_tags[:-1]\n",
    "            if len(sentence) != len(sentence_tags):\n",
    "                print(\"ERROR DIFFERENT LENGTH - at ID \"+str(row['ID'])+\" with:\")\n",
    "                print(len(sentence), '-', sentence)\n",
    "                print(len(sentence_tags), '-', sentence_tags)\n",
    "\n",
    "        # Reset vars for each sentence\n",
    "        entity = ''\n",
    "        keyword = ''\n",
    "\n",
    "        # Loop over sentence tags\n",
    "        for idx_tag, tag in enumerate(sentence_tags):\n",
    "            # If it starts a new entity\n",
    "            if tag[0] == 'B':\n",
    "                # If the previous token is an entity itself\n",
    "                if entity != '' and keyword != '':\n",
    "                    # Add to set\n",
    "                    #entity_dict[entity].add(clean_keyword(keyword))\n",
    "                    entity_dict[entity].add(re.sub(r'^[\\W\\-_]+|[\\W\\-_]+$', '', clean_keyword(keyword)))\n",
    "                # Store new entity and verify if it's composed of more word-tokens\n",
    "                entity = tag.split('-')[1]\n",
    "                keyword = sentence[idx_tag]\n",
    "\n",
    "            # If it continues an existing entity\n",
    "            elif tag[0] == 'I':\n",
    "                # If there isn't a starting token\n",
    "                if entity == '' or tag.split('-')[1] != entity:\n",
    "                    \"\"\"print('\\tERROR AT ID {0} with entity \\'{1}\\':'.format(row['ID'], entity))\n",
    "                    print('\\t\\t', ' '.join(sentence))\n",
    "                    print('\\t\\t', ' '.join(sentence_tags))\"\"\"\n",
    "                    entity = tag.split('-')[1]\n",
    "                    \"\"\"print('\\t\\t --> keep \\'{0}\\''.format(entity))\"\"\"\n",
    "                # Keep storing the entity and verify if it's composed of more word-tokens\n",
    "                keyword += ' ' + sentence[idx_tag]\n",
    "\n",
    "            # If it's not an entity\n",
    "            elif tag[0] == 'O':\n",
    "                # If the previous token is an entity\n",
    "                if entity != '' and keyword != '':\n",
    "                    # Add to set\n",
    "                    #entity_dict[entity].add(clean_keyword(keyword))\n",
    "                    entity_dict[entity].add(re.sub(r'^[\\W\\-_]+|[\\W\\-_]+$', '', clean_keyword(keyword)))\n",
    "                    # Reset vars\n",
    "                    entity = ''\n",
    "                    keyword = ''\n",
    "\n",
    "        # If the last token is an entity\n",
    "        if entity != '' and keyword != '':\n",
    "            # Add to set\n",
    "            #entity_dict[entity].add(clean_keyword(keyword))\n",
    "            entity_dict[entity].add(re.sub(r'^[\\W\\-_]+|[\\W\\-_]+$', '', clean_keyword(keyword)))\n",
    "\n",
    "    return {key: list(value) for key, value in entity_dict.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93db2887",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-30T13:22:01.127167Z",
     "iopub.status.busy": "2023-10-30T13:22:01.126752Z",
     "iopub.status.idle": "2023-10-30T13:23:39.994513Z",
     "shell.execute_reply": "2023-10-30T13:23:39.993234Z"
    },
    "papermill": {
     "duration": 98.891326,
     "end_time": "2023-10-30T13:23:39.997298",
     "exception": false,
     "start_time": "2023-10-30T13:22:01.105972",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame(total_data)\n",
    "\n",
    "# Convert the 'date' column to datetime format\n",
    "df['date'] = pd.to_datetime(df['date'])\n",
    "\n",
    "# Format the 'date' column as MM-DD-YYYY\n",
    "df['date'] = df['date'].dt.strftime('%m-%d-%Y')\n",
    "\n",
    "print('Starting extraction')\n",
    "\n",
    "# Extract entities\n",
    "df['entities'] = df.apply(extract_entities, axis=1)\n",
    "\n",
    "print('Saving results')\n",
    "\n",
    "# Save the result\n",
    "df.to_json('REP_extracted_entities.json', orient='records', indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9150a5d9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-30T13:23:40.049901Z",
     "iopub.status.busy": "2023-10-30T13:23:40.049505Z",
     "iopub.status.idle": "2023-10-30T13:23:40.055124Z",
     "shell.execute_reply": "2023-10-30T13:23:40.053980Z"
    },
    "papermill": {
     "duration": 0.034941,
     "end_time": "2023-10-30T13:23:40.057586",
     "exception": false,
     "start_time": "2023-10-30T13:23:40.022645",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def defang_iocs_in_text(text):\n",
    "    defanged_text = ioc_fanger.defang(text)\n",
    "    return defanged_text\n",
    "\n",
    "def fang_iocs_in_text(text):\n",
    "    fanged_text = ioc_fanger.fang(text)\n",
    "    return fanged_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "591f4018",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-30T13:23:40.110993Z",
     "iopub.status.busy": "2023-10-30T13:23:40.110097Z",
     "iopub.status.idle": "2023-10-30T13:23:40.116831Z",
     "shell.execute_reply": "2023-10-30T13:23:40.115887Z"
    },
    "papermill": {
     "duration": 0.036438,
     "end_time": "2023-10-30T13:23:40.119291",
     "exception": false,
     "start_time": "2023-10-30T13:23:40.082853",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def is_valid_ip(ip_str):\n",
    "    try:\n",
    "        ipaddress.ip_address(ip_str)\n",
    "        return True\n",
    "    except ValueError:\n",
    "        return False\n",
    "    \n",
    "def is_numeric(input_str):\n",
    "    pattern = r'^[\\d.,]+$'  # Pattern for a combination of numbers and dots/commas\n",
    "    return bool(re.match(pattern, input_str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b89cce3e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-30T13:23:40.171597Z",
     "iopub.status.busy": "2023-10-30T13:23:40.171098Z",
     "iopub.status.idle": "2023-10-30T13:25:32.729336Z",
     "shell.execute_reply": "2023-10-30T13:25:32.728249Z"
    },
    "papermill": {
     "duration": 112.587564,
     "end_time": "2023-10-30T13:25:32.732163",
     "exception": false,
     "start_time": "2023-10-30T13:23:40.144599",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "entity_dict = {\n",
    "        \"ACT\": set(),\n",
    "        \"APT\": set(),\n",
    "        \"DOM\": set(),\n",
    "        \"EMAIL\": set(),\n",
    "        \"ENCR\": set(),\n",
    "        \"FILE\": set(),\n",
    "        \"IDTY\": set(),\n",
    "        \"IP\": set(),\n",
    "        \"LOC\": set(),\n",
    "        \"MAL\": set(),\n",
    "        \"MD5\": set(),\n",
    "        \"OS\": set(),\n",
    "        \"PROT\": set(),\n",
    "        \"SECTEAM\": set(),\n",
    "        \"SHA1\": set(),\n",
    "        \"SHA2\": set(),\n",
    "        \"TIME\": set(),\n",
    "        \"TOOL\": set(),\n",
    "        \"URL\": set(),\n",
    "        \"VULID\": set(),\n",
    "        \"VULNAME\": set()\n",
    "    }\n",
    "\n",
    "complete_list = set()\n",
    "\n",
    "for _, r in df.iterrows():\n",
    "    for e, l in r['entities'].items():\n",
    "        for i in l:\n",
    "            if i == '':\n",
    "                continue  \n",
    "            fang_i = fang_iocs_in_text(i).lower()\n",
    "            if not is_valid_ip(fang_i) and is_numeric(fang_i):  # Avoid to ad pure numbers\n",
    "                continue\n",
    "            entity_dict[e].add(fang_i)\n",
    "            complete_list.add(fang_i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cfc4a0a6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-30T13:25:32.786260Z",
     "iopub.status.busy": "2023-10-30T13:25:32.785577Z",
     "iopub.status.idle": "2023-10-30T13:25:33.899725Z",
     "shell.execute_reply": "2023-10-30T13:25:33.898728Z"
    },
    "papermill": {
     "duration": 1.143637,
     "end_time": "2023-10-30T13:25:33.902368",
     "exception": false,
     "start_time": "2023-10-30T13:25:32.758731",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "tags = [\"APT\", \"SECTEAM\", \"IDTY\", \"OS\", \"EMAIL\", \"LOC\", \"TIME\", \"IP\", \"DOM\", \"URL\", \"PROT\", \n",
    "         \"FILE\", \"TOOL\", \"MD5\", \"SHA1\", \"SHA2\", \"MAL\", \"ENCR\", \"VULNAME\", \"VULID\", \"ACT\"]\n",
    "\n",
    "for t in tags:\n",
    "    file_path = t + \"_list.txt\"\n",
    "\n",
    "    # Open the file for writing ('w' mode)\n",
    "    with open(file_path, 'w', encoding='utf-8') as file:\n",
    "        file.write('\\n'.join(sorted(entity_dict[t])))\n",
    "        \n",
    "file_path = \"complete_list.txt\"\n",
    "# Open the file for writing ('w' mode)\n",
    "with open(file_path, 'w', encoding='utf-8') as file:\n",
    "    file.write('\\n'.join(sorted(complete_list)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 260.382513,
   "end_time": "2023-10-30T13:25:36.648166",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-10-30T13:21:16.265653",
   "version": "2.4.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}