evaluation/02_PrepareData.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json\n",
    "import matplotlib.pyplot as plt\n",
    "import itertools\n",
    "import datetime\n",
    "# Maybe use https://gitlab.com/mike01/pypacker or https://stackoverflow.com/questions/30826123/python-scapy-vs-dpkt\n",
    "import scapy\n",
    "import pickle\n",
    "import subprocess\n",
    "from subprocess import PIPE\n",
    "from pathlib import Path\n",
    "from datetime import datetime, timedelta\n",
    "import networkx as nx\n",
    "import sharedVariables\n",
    "from sharedVariables import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Variables\n",
    "directory = \"data\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "c:\\Develop\\GitHub\\DSPJ2021\\syncmesh\\evaluation\\data\\data-advanced-mongo-with-latency-12\\20220205-20-experiment-advanced-mongo-with-latency-12\\10.0.0.2.pcap\n",
      "c:\\Develop\\GitHub\\DSPJ2021\\syncmesh\\evaluation\\data\\data-advanced-mongo-with-latency-12\\20220205-20-experiment-advanced-mongo-with-latency-12\\10.0.0.3.pcap\n",
      "c:\\Develop\\GitHub\\DSPJ2021\\syncmesh\\evaluation\\data\\data-advanced-mongo-with-latency-12\\20220205-20-experiment-advanced-mongo-with-latency-12\\10.1.0.11.pcap\n",
      "c:\\Develop\\GitHub\\DSPJ2021\\syncmesh\\evaluation\\data\\data-advanced-mongo-with-latency-12\\20220205-20-experiment-advanced-mongo-with-latency-12\\10.10.0.110.pcap\n",
      "c:\\Develop\\GitHub\\DSPJ2021\\syncmesh\\evaluation\\data\\data-advanced-mongo-with-latency-12\\20220205-20-experiment-advanced-mongo-with-latency-12\\10.11.0.111.pcap\n",
      "c:\\Develop\\GitHub\\DSPJ2021\\syncmesh\\evaluation\\data\\data-advanced-mongo-with-latency-12\\20220205-20-experiment-advanced-mongo-with-latency-12\\10.12.0.112.pcap\n",
      "c:\\Develop\\GitHub\\DSPJ2021\\syncmesh\\evaluation\\data\\data-advanced-mongo-with-latency-12\\20220205-20-experiment-advanced-mongo-with-latency-12\\10.2.0.12.pcap\n",
      "c:\\Develop\\GitHub\\DSPJ2021\\syncmesh\\evaluation\\data\\data-advanced-mongo-with-latency-12\\20220205-20-experiment-advanced-mongo-with-latency-12\\10.3.0.13.pcap\n",
      "c:\\Develop\\GitHub\\DSPJ2021\\syncmesh\\evaluation\\data\\data-advanced-mongo-with-latency-12\\20220205-20-experiment-advanced-mongo-with-latency-12\\10.4.0.14.pcap\n",
      "c:\\Develop\\GitHub\\DSPJ2021\\syncmesh\\evaluation\\data\\data-advanced-mongo-with-latency-12\\20220205-20-experiment-advanced-mongo-with-latency-12\\10.5.0.15.pcap\n",
      "c:\\Develop\\GitHub\\DSPJ2021\\syncmesh\\evaluation\\data\\data-advanced-mongo-with-latency-12\\20220205-20-experiment-advanced-mongo-with-latency-12\\10.6.0.16.pcap\n",
      "c:\\Develop\\GitHub\\DSPJ2021\\syncmesh\\evaluation\\data\\data-advanced-mongo-with-latency-12\\20220205-20-experiment-advanced-mongo-with-latency-12\\10.7.0.17.pcap\n",
      "c:\\Develop\\GitHub\\DSPJ2021\\syncmesh\\evaluation\\data\\data-advanced-mongo-with-latency-12\\20220205-20-experiment-advanced-mongo-with-latency-12\\10.8.0.18.pcap\n",
      "c:\\Develop\\GitHub\\DSPJ2021\\syncmesh\\evaluation\\data\\data-advanced-mongo-with-latency-12\\20220205-20-experiment-advanced-mongo-with-latency-12\\10.9.0.19.pcap\n",
      "c:\\Develop\\GitHub\\DSPJ2021\\syncmesh\\evaluation\\data\\special-syncmesh\\20220204-22-all-functions\\10.0.0.2.pcap\n",
      "c:\\Develop\\GitHub\\DSPJ2021\\syncmesh\\evaluation\\data\\special-syncmesh\\20220204-22-all-functions\\10.1.0.11.pcap\n",
      "c:\\Develop\\GitHub\\DSPJ2021\\syncmesh\\evaluation\\data\\special-syncmesh\\20220204-22-all-functions\\10.2.0.12.pcap\n",
      "c:\\Develop\\GitHub\\DSPJ2021\\syncmesh\\evaluation\\data\\special-syncmesh\\20220204-22-all-functions\\10.3.0.13.pcap\n"
     ]
    }
   ],
   "source": [
    "# Transform PCAP to CSV\n",
    "overwrite = False\n",
    "for (root, dirnames, filenames)in os.walk(directory):\n",
    "    for filename in filenames:\n",
    "        if filename.endswith(\".pcap\") and (overwrite or (filename + \".csv\") not in filenames):\n",
    "            pcapfile = os.path.join(Path.cwd(), root, filename)\n",
    "            pcapcsvfile = os.path.join(Path.cwd(), root, filename + \".csv\")\n",
    "            print(pcapfile)\n",
    "            command = f\"tshark -r {pcapfile} -T fields -Eheader=y -Eseparator=, -e frame.time_epoch -e tcp.analysis.initial_rtt -e ip.id -e ip.src -e ip.dst -e ip.len -e tcp.srcport -e tcp.dstport -Y \\\"(not ip.addr == 10.1.0.255) and (not tcp.srcport == 22) and (not ip.addr == 10.62.0.0/16)\\\" > {pcapcsvfile}\"\n",
    "            # print(command)\n",
    "            result = subprocess.run(command, shell=True, stdout=PIPE, stderr=PIPE, universal_newlines=True)\n",
    "            # print(result.returncode, result.stdout, result.stderr)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "9518128f597d7b00dc14729602cfd87fb7b2cf75925976bcb0d0e328a830a12b"
  },
  "kernelspec": {
   "display_name": "Python 3.9.6 64-bit",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}