tokenize.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import regex as re\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import highlighting_functions as hf\n",
    "from importlib import reload\n",
    "\n",
    "_ = reload(hf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fun(t):\n",
    "    t = t.copy()\n",
    "    for i in range(len(t)):\n",
    "        t[i] += \"a\"\n",
    "    return t\n",
    "\n",
    "\n",
    "a = [\"a\", \"b\", \"c\"]\n",
    "b = fun(a)\n",
    "print(a)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_file = \"input.py\"\n",
    "\n",
    "with open(input_file) as f:\n",
    "    text = f.read()\n",
    "\n",
    "print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens, tags = hf.tokenize(text)\n",
    "print(tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## tagging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"# tokens\", len(tokens))\n",
    "\n",
    "known_default = {\n",
    "    \"assign\": [\"=\", \"<-\"],\n",
    "    \"punct\": [\",\", \";\", \".\"],\n",
    "    \"op\": r\"!%&/+-*:<>^\",\n",
    "    \"brac_op\": r\"([{\",\n",
    "    \"brac_cl\": r\")]}\",\n",
    "    \"keyword\": [\n",
    "        \"for\",\n",
    "        \"while\",\n",
    "        \"foreach\",\n",
    "        \"as\",\n",
    "        \"in\",\n",
    "        \"if\",\n",
    "        \"else\",\n",
    "        \"elif\",\n",
    "        \"and\",\n",
    "        \"or\",\n",
    "        \"not\",\n",
    "        \"return\",\n",
    "    ],\n",
    "}\n",
    "\n",
    "\n",
    "# first: tag individual tokens\n",
    "tags = hf.tag_individuals(tokens, tags, known_default)\n",
    "\n",
    "\n",
    "# merge and tag again to catch multiple character assignment etc.\n",
    "tokens, tags = hf.merge_adjacent(tokens, tags, known_default)\n",
    "\n",
    "# rename brackets\n",
    "tags, brac_level = hf.bracket_levels(tags)\n",
    "print(tokens)\n",
    "print(tags)\n",
    "# second: context\n",
    "\n",
    "tags = hf.tag_functions(tokens, tags)\n",
    "tags = hf.tag_variables(tokens, tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for x in zip(tags, tokens):\n",
    "    if x[0] == \"unk\":\n",
    "        print(\"\\t\".join(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_names = (\n",
    "    \"x\",\n",
    "    \"x\",\n",
    "    \"before_.\",\n",
    "    \"after_.\",\n",
    "    \"before_(\",\n",
    "    \"before_[\",\n",
    "    \"before_op\",\n",
    "    \"after_op\",\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## comments?\n",
    "\n",
    "- many unknowns\n",
    "- Starts with token not present in completely tagged lines\n",
    "- comment symbol\n",
    "    - not after assign\n",
    "    - not inside brackets\n",
    "    - NOT necessatily start of line"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lines = []\n",
    "lines_tags = []\n",
    "\n",
    "current_line = []\n",
    "current_line_tags = []\n",
    "for i, token in enumerate(tokens):\n",
    "    if \"\\n\" in token:\n",
    "        lines.append(current_line)\n",
    "        lines_tags.append(current_line_tags)\n",
    "        current_line = []\n",
    "        current_line_tags = []\n",
    "    else:\n",
    "        current_line.append(token)\n",
    "        current_line_tags.append(tags[i])\n",
    "\n",
    "unks_per_line = [np.array(list([t == \"var\"] for t in L)).sum() for L in lines]\n",
    "unks_per_line"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = np.zeros((len(tokens), len(feature_names)))\n",
    "\n",
    "print(\"feature matrix\", features.shape)\n",
    "print(*zip(range(len(feature_names)), feature_names))\n",
    "\n",
    "# in brackets?\n",
    "features[:, 1] = np.array(brac_level)\n",
    "\n",
    "# before and after?\n",
    "for i, token in enumerate(tokens):\n",
    "    if token == \".\":\n",
    "        if i > 0:\n",
    "            features[i - 1, 2] = 1\n",
    "        if i < len(tokens) - 1:\n",
    "            features[i + 1, 3] = 1\n",
    "\n",
    "    if token == \"(\":\n",
    "        if i > 0:\n",
    "            features[i - 1, 4] = 1\n",
    "\n",
    "    if token == \"[\":\n",
    "        if i > 0:\n",
    "            features[i - 1, 5] = 1\n",
    "\n",
    "    if tags[i] == \"op\":\n",
    "        if i > 0:\n",
    "            features[i - 1, 6] = 1\n",
    "        if i < len(tokens) - 1:\n",
    "            features[i + 1, 7] = 1\n",
    "\n",
    "\n",
    "plt.figure()\n",
    "plt.imshow(features, cmap=\"gray\")\n",
    "plt.title(\"feature matrix\")\n",
    "plt.xlabel(\"feature\")\n",
    "plt.ylabel(\"token\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "\n",
    "rng = np.random.default_rng()\n",
    "\n",
    "pca = PCA(n_components=2)\n",
    "\n",
    "f_PCA = pca.fit_transform(features)\n",
    "print(f_PCA.shape)\n",
    "\n",
    "plt.figure()\n",
    "plt.scatter(x=f_PCA[:, 0], y=f_PCA[:, 1], alpha=0.1)\n",
    "for i, t in enumerate(tokens):\n",
    "    c = \"#000\"\n",
    "    if tags[i] == \"unk\":\n",
    "        c = \"r\"\n",
    "    elif tags[i] == \"num\":\n",
    "        c = \"g\"\n",
    "\n",
    "    plt.text(x=f_PCA[i, 0], y=f_PCA[i, 1], s=repr(t), rotation=45, color=c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = tuple(sorted(set(tags)))\n",
    "print(classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## cleanup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"# tokens/tags\", len(tokens))\n",
    "\n",
    "# Merge adjacent of same class\n",
    "tokens, tags = hf.merge_adjacent(tokens, tags)\n",
    "\n",
    "print(\"after merge # tokens/tags\", len(tokens))\n",
    "print(\"count unknown\", sum([t == \"unk\" for t in tags]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "EXCLUDE_TAGS = (\"unk\", \"wsp\")\n",
    "html_text = hf.tokens_to_html(tokens, tags, EXCLUDE_TAGS)\n",
    "\n",
    "print(html_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "css_path = \"_style.css\"\n",
    "css_link = f'<link rel=\"stylesheet\" type=\"text/css\" href=\"{css_path}\">'\n",
    "final_html = f\"\"\"\n",
    "<head>\n",
    "    {css_link}\n",
    "</head>\n",
    "<body>\n",
    "    {html_text}\n",
    "</body>\n",
    "\"\"\"\n",
    "\n",
    "with open(\"output.html\", \"w\") as f:\n",
    "    f.write(final_html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test method\n",
    "\n",
    "final_html, classes = hf.highlight_code(text)\n",
    "with open(\"output.html\", \"w\") as f:\n",
    "    f.write(final_html)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "textapp",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}