標註資料用_大量.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "刑事判決_105,中交簡,1002_2016-04-25.json\n"
     ]
    }
   ],
   "source": [
    "import os, json\n",
    "import pandas as pd\n",
    "import re\n",
    "\n",
    "train_num = 103000\n",
    "\n",
    "path_to_json = 'J:\\Legal_Judgment\\臺灣新北地方法院_改_V3'\n",
    "json_files = [pos_json for pos_json in os.listdir(path_to_json) if pos_json.endswith('.json')]\n",
    "print(json_files[0])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [],
   "source": [
    "with open('J://Legal_Judgment//臺灣新北地方法院_改_V3//'+json_files[0],\"r\",encoding=\"utf-8\") as f:\n",
    "    dt = json.load(f)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [],
   "source": [
    "dic = {\n",
    "    \"id\": 0,\n",
    "    \"court\": [dt[\"court\"]],\n",
    "    \"date\": [dt[\"date\"]],\n",
    "    \"no\": [dt[\"no\"]],\n",
    "    \"sys\": [dt[\"sys\"]],\n",
    "    \"reason\": [dt[\"reason\"]],\n",
    "    \"type\": [dt[\"type\"]],\n",
    "    \"mainText\": [dt[\"mainText\"]],\n",
    "    \"judgement\": [dt[\"judgement\"]],\n",
    "    #\"judgement_SIT\": [dt[\"judgement_SIT\"]],\n",
    "    #\"Situation\": [dt[\"Situation\"]]\n",
    "}\n",
    "df = pd.DataFrame(dic)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000\n",
      "2000\n",
      "3000\n",
      "4000\n",
      "5000\n",
      "6000\n",
      "7000\n",
      "8000\n",
      "9000\n",
      "10000\n",
      "11000\n",
      "12000\n",
      "13000\n",
      "14000\n",
      "15000\n",
      "16000\n",
      "17000\n",
      "18000\n",
      "19000\n",
      "20000\n",
      "21000\n",
      "22000\n",
      "23000\n",
      "24000\n",
      "25000\n",
      "26000\n",
      "27000\n",
      "28000\n",
      "29000\n",
      "30000\n",
      "31000\n",
      "32000\n",
      "33000\n",
      "34000\n",
      "35000\n",
      "36000\n",
      "37000\n",
      "38000\n",
      "39000\n",
      "40000\n",
      "41000\n",
      "42000\n",
      "43000\n",
      "44000\n",
      "45000\n",
      "46000\n",
      "47000\n",
      "48000\n",
      "49000\n",
      "50000\n",
      "51000\n",
      "52000\n",
      "53000\n",
      "54000\n",
      "55000\n",
      "56000\n",
      "57000\n",
      "58000\n",
      "59000\n",
      "60000\n",
      "61000\n",
      "62000\n",
      "63000\n",
      "64000\n",
      "65000\n",
      "66000\n",
      "67000\n",
      "68000\n",
      "69000\n",
      "70000\n",
      "71000\n",
      "72000\n",
      "73000\n",
      "74000\n",
      "75000\n",
      "76000\n",
      "77000\n",
      "78000\n",
      "79000\n",
      "80000\n",
      "81000\n",
      "82000\n",
      "83000\n",
      "84000\n",
      "85000\n",
      "86000\n",
      "87000\n",
      "88000\n",
      "89000\n",
      "90000\n",
      "91000\n",
      "92000\n",
      "93000\n",
      "94000\n",
      "95000\n",
      "96000\n",
      "97000\n",
      "98000\n",
      "99000\n",
      "100000\n",
      "101000\n",
      "102000\n",
      "103000\n"
     ]
    }
   ],
   "source": [
    "#my_regex = \"\\（.*?\\）|\\(([^\\)0-9零一二三四五六七八九十]?|[^\\).]{2,})\\)\"\n",
    "\n",
    "#spe = '丙○○意圖為自己不法之所有，基於搶奪之犯意，先後為下列搶奪行為：(一)於民國105年10月8日晚間7時20分許，騎乘登記於不知情之王介平名下之車牌號碼000-000號普通重型機車，行經高雄市○○區○○○路000號前時，將口罩脫下遮掩車牌，自後靠近步行在人行道上之甲○○，趁甲○○不及防備之際，以左手徒手掠取甲○○右手持有之背包1只（內有現金新臺幣【下同】9500元、索尼牌ZR綠色及Xperformance金色行動電話共2具【IMEI：000000000000000、000000000000000號】、長夾1個、信用卡及金融卡共5張、國民身分證2張、國民健康保險卡1張、普通重型機車駕照1張、識別證2張等物）得手後逃逸。嗣因丙○○將背包內之現金取走後，將背包棄置在高雄市鳳山區武營路某處，再經民眾在高雄市鳳山區衛武營附近拾獲上開背包（內有長夾1個、國民身分證2張、國民健康保險卡1張、信用卡3張、金融卡1張等物），遂送交警察機關發還甲○○領回。(二)於同年10月10日晚間11時5分許，騎乘向不知情之劉伊恩借用之車牌號碼000-0000號普通重型機車，行經高雄市○○區○○○路000號前時，使用口罩遮掩車牌，自後靠近步行在人行道上之少年乙○○（89年生，基於兒童及少年福利與權益保障法第69條第2項規定不得揭露足以識別少年身分之資訊，年籍詳卷），趁乙○○不及防備之際，以左手徒手掠取乙○○右手持有之背包1只（內有皮夾1只、現金3萬2000元、提款卡1張、國民身分證1張、國民健康保險卡1張等物）得手後逃逸，並將現金取出，其餘物品則丟棄。嗣為警循線於105年10月16日凌晨2時許，在高雄市○○區○○路000巷00號前查獲，並扣得現金2600元。'\n",
    "show = 0\n",
    "cnt = 0\n",
    "for x in range(1,len(json_files)):\n",
    "    with open('J://Legal_Judgment//臺灣新北地方法院_改_V3//'+json_files[x],\"r\",encoding=\"utf-8\") as f:\n",
    "        dt = json.load(f)\n",
    "\n",
    "    try:\n",
    "        #if (str(dt[\"judgement\"]).index(str(dt[\"Situation\"])) < 0):\n",
    "            #print(\"error1:\" + dt[\"no\"])\n",
    "            #continue\n",
    "\n",
    "\n",
    "        new = pd.DataFrame({\n",
    "        \"id\" : x,\n",
    "        \"court\": [dt[\"court\"]],\n",
    "        \"date\": [dt[\"date\"]],\n",
    "        \"no\": [dt[\"no\"]],\n",
    "        \"sys\": [dt[\"sys\"]],\n",
    "        \"reason\": [dt[\"reason\"]],\n",
    "        \"type\": [dt[\"type\"]],\n",
    "        \"mainText\": [dt[\"mainText\"]],\n",
    "        \"judgement\": [dt[\"judgement\"]],\n",
    "        #\"judgement_SIT\": [dt[\"judgement_SIT\"]],\n",
    "        #\"Situation\": [dt[\"Situation\"]]\n",
    "        })\n",
    "\n",
    "        bol = 0\n",
    "        show = 0\n",
    "\n",
    "        if (bol == 1):\n",
    "            continue\n",
    "\n",
    "        df = df.append(new)\n",
    "        cnt += 1\n",
    "        if (cnt % 10000 == 0):\n",
    "            print(cnt)\n",
    "\n",
    "    except:\n",
    "        print(dt[\"no\"])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [
    {
     "data": {
      "text/plain": "id           103066\ncourt        103066\ndate         103066\nno           103066\nsys          103066\nreason       103066\ntype         103066\nmainText     103066\njudgement    103066\ndtype: int64"
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.count()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [],
   "source": [
    "df.to_csv('J://Legal_Judgment//Result_10W.csv', index=0)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [
    {
     "data": {
      "text/plain": "       court                       date            no sys reason type  \\\nid                                                                      \n0   臺灣臺中地方法院  2016-04-25T00:00:00+08:00  105,中交簡,1002  刑事   公共危險   判決   \n1   臺灣臺中地方法院  2016-04-29T00:00:00+08:00  105,中交簡,1003  刑事   公共危險   判決   \n\n                                             mainText  \\\nid                                                      \n0   周文同駕駛動力交通工具而吐氣所含酒精濃度達每公升零點二五毫克以上，處有期徒刑貳月，如易科罰金...   \n1   江孝文駕駛動力交通工具而吐氣所含酒精濃度達每公升零點二五毫克以上，處有期徒刑貳月，如易科罰金...   \n\n                                            judgement  \nid                                                     \n0   臺灣臺中地方法院刑事簡易判決105年度中交簡字第1002號聲請人臺灣臺中地方法院檢察署檢察官...  \n1   臺灣臺中地方法院刑事簡易判決105年度中交簡字第1003號公訴人臺灣臺中地方法院檢察署檢察官...  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>court</th>\n      <th>date</th>\n      <th>no</th>\n      <th>sys</th>\n      <th>reason</th>\n      <th>type</th>\n      <th>mainText</th>\n      <th>judgement</th>\n    </tr>\n    <tr>\n      <th>id</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <td>0</td>\n      <td>臺灣臺中地方法院</td>\n      <td>2016-04-25T00:00:00+08:00</td>\n      <td>105,中交簡,1002</td>\n      <td>刑事</td>\n      <td>公共危險</td>\n      <td>判決</td>\n      <td>周文同駕駛動力交通工具而吐氣所含酒精濃度達每公升零點二五毫克以上，處有期徒刑貳月，如易科罰金...</td>\n      <td>臺灣臺中地方法院刑事簡易判決105年度中交簡字第1002號聲請人臺灣臺中地方法院檢察署檢察官...</td>\n    </tr>\n    <tr>\n      <td>1</td>\n      <td>臺灣臺中地方法院</td>\n      <td>2016-04-29T00:00:00+08:00</td>\n      <td>105,中交簡,1003</td>\n      <td>刑事</td>\n      <td>公共危險</td>\n      <td>判決</td>\n      <td>江孝文駕駛動力交通工具而吐氣所含酒精濃度達每公升零點二五毫克以上，處有期徒刑貳月，如易科罰金...</td>\n      <td>臺灣臺中地方法院刑事簡易判決105年度中交簡字第1003號公訴人臺灣臺中地方法院檢察署檢察官...</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('J://Legal_Judgment//Result_10W.csv', index_col=0)\n",
    "df.head(2)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [],
   "source": [
    "df['reason'] = df['reason'].str.replace('等','')\n",
    "df['reason'] = df['reason'].str.replace('罪','')\n",
    "df['reason'] = df['reason'].str.replace(\"條例\",\"\")\n",
    "df['reason'] = df['reason'].str.replace(\"防制\",\"\")\n",
    "df['reason'] = df['reason'].str.replace(\"管制\",\"\")\n",
    "df['reason'] = df['reason'].str.replace(\"違反\",\"\")\n",
    "df['reason'] = df['reason'].str.replace(\"違法著作權\",\"著作權法\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [
    {
     "data": {
      "text/plain": "毒品危害          25870\n公共危險          17444\n竊盜            10539\n過失傷害           9894\n詐欺             8052\n              ...  \n家庭暴力之妨害性自主        1\n家暴竊盜              1\n交付法庭錄音光碟          1\n重傷害致死             1\n人工生殖法             1\nName: reason, Length: 267, dtype: int64"
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[:,'reason'].value_counts()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [],
   "source": [
    "my_regex = r\"[\\(\\（]([^一二三四五六七八九零十1234567890]{1}[^\\)\\）]+|[一二三四五六七八九零十1234567890]{1}[^\\)\\）]{1,})[\\)\\）]\"\n",
    "df['judgement'] = df['judgement'].str.replace(my_regex, \"\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "outputs": [],
   "source": [
    "class Data(object):\n",
    "    def __init__(self):\n",
    "        self.paragraphs = []\n",
    "        self.id = \"0\"\n",
    "        self.title = \"\"\n",
    "\n",
    "class Djson(object):\n",
    "    def __init__(self):\n",
    "        self.version = \"v1.0\"\n",
    "        self.data = []"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [],
   "source": [
    "djson1 = ''\n",
    "djson1 = Djson()\n",
    "i = 0\n",
    "for index, row in df.iterrows():\n",
    "    try:\n",
    "        d = Data()\n",
    "        d.paragraphs = [{\"id\" : \"TRAIN_\" + str(index),\n",
    "                      #\"context\" : row.reason + \"[SEP]\" + row.judgement,\n",
    "                      \"context\" : row.judgement,\n",
    "                      \"qas\":[{\n",
    "                          \"question\" :\"?\",\n",
    "                          \"id\" : \"TRAIN_\" + str(index) + \"_QUERY_0\",\n",
    "                          \"answers\" :[{\n",
    "                              \"text\" : \"NoAnswer?\",\n",
    "                              \"answer_start\" : 0\n",
    "                          }]\n",
    "                      }]\n",
    "                    }\n",
    "                    ]\n",
    "        d.id = \"TRAIN_\" + str(index)\n",
    "        d.title = row.no\n",
    "        djson1.data.append(d)\n",
    "\n",
    "        if (index / 10000 > i):\n",
    "            i += 1\n",
    "            y = json.dumps(djson1, default=lambda o: o.__dict__, ensure_ascii=False)\n",
    "            with open('./QA_Test1/HBert/BigData2/Big10W_' + str(i) +'_X.json', 'w', encoding='utf8') as outfile:\n",
    "                json.dump(djson1, outfile, default=lambda o: o.__dict__ , ensure_ascii=False)\n",
    "            djson1 = Djson()\n",
    "\n",
    "    except:\n",
    "        print(index)\n",
    "        print(row.no)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15\n"
     ]
    }
   ],
   "source": [
    "str = '0000000-0000000'\n",
    "print(len(str))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}