Convert_to_JSON_IOB_Selected_Entities.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "from tqdm import tqdm \n",
    "from collections import defaultdict\n",
    "\n",
    "root_path ='/home/santosh/Work/Datasets/Datasets/Covid-19pubmed/'\n",
    "\n",
    "annotation_path = root_path+'annotations/'\n",
    "text_path = root_path+'raw/'\n",
    "\n",
    "# annotation_files = sorted(glob.glob(annotation_path + '*.a*'))\n",
    "text_files = sorted(glob.glob(text_path + '*.txt*'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load sci_spacy\n",
    "# https://allenai.github.io/scispacy/\n",
    "\n",
    "import scispacy\n",
    "import spacy\n",
    "# import en_core_web_sm\n",
    "import en_core_sci_sm\n",
    "nlp = en_core_sci_sm.load() # for sentencising. The best sentenciser for biomedical text\n",
    "# nlp = en_core_web_sm.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 17740/17740 [09:20<00:00, 31.63it/s]\n"
     ]
    }
   ],
   "source": [
    "# get datasets for different entity type as well as datasets with all the entities in one.\n",
    "non_overlapping_dataset = defaultdict(list) # the entities are non overlapping here\n",
    "# overlapping_dataset = defaultdict(list) \n",
    "# Disorder_dataset= defaultdict(list) #(DISO)\n",
    "# Species_dataset= defaultdict(list) #(SPEC)\n",
    "Chemical_Drug_dataset= defaultdict(list) #(CHED)\n",
    "# Gene_Protein_dataset = defaultdict(list)#(PRGE)\n",
    "# Enzyme_dataset = defaultdict(list)#(ENZY)\n",
    "# Anatomy_dataset = defaultdict(list)#(ANAT)\n",
    "Biological_Process_dataset = defaultdict(list)#(PROC)\n",
    "# Molecular_Function_dataset = defaultdict(list)#(FUNC)\n",
    "# Cellular_Component_dataset = defaultdict(list)#(COMP)\n",
    "# Pathway_dataset = defaultdict(list)#(PATH)\n",
    "# microRNA_dataset = defaultdict(list) #(MRNA)\n",
    "\n",
    "# iter = 0\n",
    "for each_text_path in tqdm(text_files):\n",
    "\n",
    "#     iter = iter+1\n",
    "#     if iter==10:\n",
    "#         break\n",
    "    annotation_list =[] # list of entities from the annotations file\n",
    "    text_list = [] # sentences list from the text file, with sentence spans\n",
    "    non_overlapping_annotation_list = [] # list with non overlapping entities\n",
    "    \n",
    "    # read the text file \n",
    "    with open(each_text_path) as text_file:\n",
    "        text = text_file.read()\n",
    "        # sentencise the text and get the spans\n",
    "        doc = nlp(text)\n",
    "        for sent_ in doc.sents:\n",
    "            sent = str(sent_).replace('\\n',' ')\n",
    "            text_list.append([sent, sent_.start_char, sent_.end_char])\n",
    "    \n",
    "    # read the annotations for the respective raw text file\n",
    "    each_annotation_path = annotation_path+each_text_path.split('/')[-1].replace('txt','a1')\n",
    "    \n",
    "#     print(each_text_path)\n",
    "#     print(each_annotation_path)\n",
    "    \n",
    "    with open(each_annotation_path) as annotation_file:\n",
    "        annotation = annotation_file.readlines()\n",
    "    \n",
    "    # load the anotations into a list. There is a small problem here due to data formating consistencies.\n",
    "    for each_line in annotation:\n",
    "        temp_ = each_line.split()\n",
    "        if 'T' in temp_[0]:\n",
    "            if len(temp_)>5:\n",
    "                merged_anno = temp_[0:4]+ [' '.join(temp_[4:])] # because of the general split we will have to merge entities with spaces together\n",
    "                annotation_list.append(merged_anno)\n",
    "            \n",
    "            else:\n",
    "                annotation_list.append(temp_)\n",
    "\n",
    "                # Remove the overlaps\n",
    "    for inx in range(-1, len(annotation_list)):\n",
    "        if inx == len(annotation_list)-1:\n",
    "            break\n",
    "        if inx ==-1:\n",
    "            non_overlapping_annotation_list.append(annotation_list[inx+1])\n",
    "        else:\n",
    "            if annotation_list[inx+1][2]>annotation_list[inx][3]:\n",
    "                non_overlapping_annotation_list.append(annotation_list[inx+1])\n",
    "            \n",
    "\n",
    "    # we need to remove the off-set for the sentence span to reflect the span of the entities at the sentence level rather than at the document level\n",
    "    for each_annotation in annotation_list:    \n",
    "            \n",
    "        st_ann_sp = int(each_annotation[2]) #  start of annotation span\n",
    "        en_ann_sp = int(each_annotation[3]) #  end of annotation span\n",
    "        ann_type = each_annotation[1] #  annotation type\n",
    "        ann = each_annotation[4] # annotation\n",
    "            \n",
    "        for each_text in text_list:\n",
    "            \n",
    "            snt_text = str(each_text[0]) # sentence text\n",
    "            st_snt_sp = int(each_text[1]) # Start of sentence span\n",
    "            en_snt_sp = int(each_text[2]) # end of sentence span\n",
    "            \n",
    "                # check if the annotation span is with in the sentence span \n",
    "            if st_snt_sp <= st_ann_sp <= en_snt_sp and st_snt_sp <= en_ann_sp <= en_snt_sp:\n",
    "                if ann in snt_text: # process only if the annotation is in the sentence \n",
    "                    anno_list = [st_ann_sp-st_snt_sp, en_ann_sp-st_snt_sp,ann, ann_type] # get the annotation details\n",
    "                    if snt_text[st_ann_sp-st_snt_sp:en_ann_sp-st_snt_sp] == ann: # final check if annotation is indeed in the text and has the right span\n",
    "#                         if ann_type == 'SPEC':\n",
    "#                             Species_dataset[snt_text].append(anno_list)\n",
    "#                         elif ann_type == 'DISO':\n",
    "#                             Disorder_dataset[snt_text].append(anno_list)\n",
    "                        elif ann_type == 'CHED':\n",
    "                            Chemical_Drug_dataset[snt_text].append(anno_list)\n",
    "#                         elif ann_type == 'PRGE':\n",
    "#                             Gene_Protein_dataset[snt_text].append(anno_list)\n",
    "#                         elif ann_type == 'ENZY':\n",
    "#                             Enzyme_dataset[snt_text].append(anno_list)\n",
    "#                         elif ann_type == 'ANAT':\n",
    "#                             Anatomy_dataset[snt_text].append(anno_list)\n",
    "                        elif ann_type == 'PROC':\n",
    "                            Biological_Process_dataset[snt_text].append(anno_list)\n",
    "#                         elif ann_type == 'FUNC':\n",
    "#                             Molecular_Function_dataset[snt_text].append(anno_list)\n",
    "#                         elif ann_type == 'COMP':\n",
    "#                             Cellular_Component_dataset[snt_text].append(anno_list)\n",
    "#                         elif ann_type == 'PATH':\n",
    "#                             Pathway_dataset[snt_text].append(anno_list)\n",
    "#                         elif ann_type == 'MRNA':\n",
    "#                             microRNA_dataset[snt_text].append(anno_list)\n",
    "\n",
    "#                 overlapping_dataset[each_text[0]].append([int(each_annotation[2])-int(each_text[1]), int(each_annotation[3])-int(each_text[1]),each_annotation[4], each_annotation[1]])                                                        \n",
    "                \n",
    "                # check if the annotations have overlap, the entities span in sentence should be greater than the previous one\n",
    "    \n",
    "    for each_annotation in non_overlapping_annotation_list:    \n",
    "\n",
    "        st_ann_sp = int(each_annotation[2]) #  start of annotation span\n",
    "        en_ann_sp = int(each_annotation[3]) #  end of annotation span\n",
    "        ann_type = each_annotation[1] #  annotation type\n",
    "        ann = each_annotation[4] # annotation\n",
    "\n",
    "        for each_text in text_list:\n",
    "\n",
    "            snt_text = str(each_text[0]) # sentence text\n",
    "            st_snt_sp = int(each_text[1]) # Start of sentence span\n",
    "            en_snt_sp = int(each_text[2]) # end of sentence span\n",
    "            \n",
    "            if st_snt_sp <= st_ann_sp <= en_snt_sp and st_snt_sp <= en_ann_sp <= en_snt_sp:\n",
    "                if ann in snt_text: # process only if the annotation is in the sentence \n",
    "                    anno_list = [st_ann_sp-st_snt_sp, en_ann_sp-st_snt_sp,ann, ann_type] # get the annotation details\n",
    "                    if snt_text[st_ann_sp-st_snt_sp:en_ann_sp-st_snt_sp] == ann: # final check if annotation is indeed in the text and has the right span\n",
    "                        non_overlapping_dataset[snt_text].append(anno_list)\n",
    "    \n",
    "#     break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate train, test and dev pmc ids\n",
    "import math\n",
    "import random\n",
    "import jsonlines\n",
    "import os\n",
    "import pathlib\n",
    "import csv\n",
    "from nltk.tokenize import WordPunctTokenizer, wordpunct_tokenize\n",
    "\n",
    "\n",
    "def get_train_dev_test_indxs(total_num_annotations):\n",
    "\n",
    "    percentage=0.70\n",
    "    iter = 0\n",
    "\n",
    "    trainids = []\n",
    "    devids = []\n",
    "    testids =[]\n",
    "\n",
    "    nLines = total_num_annotations\n",
    "    nTrain = int(nLines*percentage) \n",
    "    nValid = math.floor((nLines - nTrain)/2)\n",
    "    nTest = nLines - (nTrain+nValid)\n",
    "\n",
    "    deck = list(range(0, nLines))\n",
    "    random.seed(45) # This will be fixed for reproducibility\n",
    "    random.shuffle(deck)\n",
    "\n",
    "    train_ids = deck[0:nTrain]\n",
    "    devel_ids = deck[nTrain:nTrain+nValid]\n",
    "    test_ids = deck[nTrain+nValid:nTrain+nValid+nTest]\n",
    "\n",
    "    return train_ids, devel_ids, test_ids\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_to_jsonl_format(dictionary_dataset, train_ids, devel_ids, test_ids,path, folder_name): \n",
    "    \n",
    "    result_path = cwd_+folder_name\n",
    "    pathlib.Path(result_path).mkdir(parents=True, exist_ok=True)\n",
    "    \n",
    "    train_jsonl_data = []\n",
    "    devel_jsonl_data=[]\n",
    "    test_jsonl_data = []\n",
    "    \n",
    "    iter = 1\n",
    "    \n",
    "    for key, values in tqdm(dictionary_dataset.items(), total=len(dictionary_dataset)):\n",
    "        \n",
    "        text = str(key)\n",
    "        entities =[]\n",
    "\n",
    "        for each_ner in values:\n",
    "            point_start = int(each_ner[0])\n",
    "            point_end = int(each_ner[1])\n",
    "            label = each_ner[3]\n",
    "            entities.append((point_start, point_end,label))\n",
    "        \n",
    "        if iter in train_ids:\n",
    "            train_jsonl_data.append((text, {\"entities\" : entities}))\n",
    "        elif iter in devel_ids:\n",
    "            devel_jsonl_data.append((text, {\"entities\" : entities}))\n",
    "        elif iter in test_ids:\n",
    "            test_jsonl_data.append((text, {\"entities\" : entities}))            \n",
    "    \n",
    "        iter = iter+1\n",
    "        \n",
    "    with jsonlines.open(result_path+'train.json', mode='w') as writer:\n",
    "        for each_line in test_jsonl_data:\n",
    "            writer.write(each_line)\n",
    "\n",
    "    with jsonlines.open(result_path+'devel.json', mode='w') as writer:\n",
    "        for each_line in devel_jsonl_data:\n",
    "            writer.write(each_line)\n",
    "            \n",
    "    with jsonlines.open(result_path+'test.json', mode='w') as writer:\n",
    "        for each_line in test_jsonl_data:\n",
    "            writer.write(each_line)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_sub_span(sub_span_range, full_spans_range):\n",
    "    # if a sub span is present in full span return it\n",
    "    if sub_span_range[0] in range(full_spans_range[0],full_spans_range[1]):\n",
    "        return sub_span_range\n",
    "\n",
    "    \n",
    "def convert2IOB(text_data, ner_tags):\n",
    "    tokenizer = WordPunctTokenizer()\n",
    "\n",
    "#     print(text_data, ner_tags)\n",
    "    tokens = []\n",
    "    ners = []\n",
    "    spans = []\n",
    "\n",
    "    split_text = tokenizer.tokenize(text_data)\n",
    "    span_text = list(tokenizer.span_tokenize(text_data))\n",
    "    # for each word token append 'O'\n",
    "    arr = ['O'] * len(split_text)\n",
    "\n",
    "    for each_tag in ner_tags:\n",
    "        span_list = (int(each_tag[0]), int(each_tag[1]))\n",
    "        token_list = wordpunct_tokenize(each_tag[2])\n",
    "        ner_list = wordpunct_tokenize(each_tag[3])\n",
    "\n",
    "        if (len(token_list) > len(ner_list)):\n",
    "            ner_list = len(token_list) * ner_list\n",
    "        for i in range(0, len(ner_list)):\n",
    "            # The logic here is look for the first B-tag and then append I-tag next\n",
    "            if (i == 0):\n",
    "                ner_list[i] = 'B-' + ner_list[i]\n",
    "            else:\n",
    "                ner_list[i] = 'I-' + ner_list[i]\n",
    "\n",
    "        tokens.append(token_list)\n",
    "        ners.append(ner_list)\n",
    "        spans.append(span_list)\n",
    "\n",
    "    split_token_span_list = list(zip(split_text, span_text))\n",
    "    span_ner_list = list(zip(spans, ners))\n",
    "\n",
    "    \n",
    "    sub_spans =[] # get sub spans from the full spans of the ner\n",
    "\n",
    "    for each_span_ner_list in span_ner_list:\n",
    "        # in full range ner e.g., [144, 150, 'COVID-19', 'DISO']\n",
    "        count = 0\n",
    "        # count is to keep track of the B, I, sub tags in the ner list\n",
    "        for each_token in split_token_span_list:\n",
    "            sub_spans_ = find_sub_span(each_token[1], each_span_ner_list[0])\n",
    "            if sub_spans_:\n",
    "                sub_spans.append([sub_spans_,each_span_ner_list[1][count]])\n",
    "                count = count+1\n",
    "            \n",
    "            \n",
    "    \n",
    "    for i, each_span_token in enumerate(split_token_span_list):\n",
    "        for each_ner_span in sub_spans:\n",
    "            if each_span_token[1] == each_ner_span[0]:\n",
    "                arr[i] = ''.join(each_ner_span[1])\n",
    "\n",
    "    return zip(split_text, arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_to_IOB_format(dictionary_dataset, train_ids, devel_ids, test_ids, path, folder_name):\n",
    "    \n",
    "    result_path = cwd_+folder_name\n",
    "    pathlib.Path(result_path).mkdir(parents=True, exist_ok=True)\n",
    "    \n",
    "    with open(result_path + 'train.csv', 'w', newline='\\n') as f1, open(result_path + 'devel.csv', 'w',\n",
    "                        newline='\\n') as f2, open(result_path + 'test.csv', 'w', newline='\\n') as f3:\n",
    "\n",
    "        train_writer = csv.writer(f1, delimiter='\\t', lineterminator='\\n')\n",
    "        dev_writer = csv.writer(f2, delimiter='\\t', lineterminator='\\n')\n",
    "        test_writer = csv.writer(f3, delimiter='\\t', lineterminator='\\n')\n",
    "        \n",
    "        iter = 0\n",
    "\n",
    "        for key, values in tqdm(dictionary_dataset.items(), total=len(dictionary_dataset)):\n",
    "            \n",
    "            text = str(key)\n",
    "#             print(text)\n",
    "            tagged_tokens = convert2IOB(text, values)\n",
    "\n",
    "\n",
    "            if iter in train_ids:\n",
    "                for each_token in tagged_tokens:\n",
    "                    train_writer.writerow(list(each_token))\n",
    "                train_writer.writerow('')\n",
    "\n",
    "            elif iter in devel_ids:\n",
    "                for each_token in tagged_tokens:\n",
    "                    dev_writer.writerow(list(each_token))\n",
    "                dev_writer.writerow('')\n",
    "\n",
    "            elif iter in test_ids:\n",
    "                for each_token in tagged_tokens:\n",
    "                    test_writer.writerow(list(each_token))\n",
    "                test_writer.writerow('')\n",
    "            \n",
    "            iter = iter+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/santosh/Work/GitHub/COVID-19-Named-Entity-Recognition-/Datasets/\n"
     ]
    }
   ],
   "source": [
    "cwd_ = os.getcwd()+'/Datasets/'\n",
    "print(cwd_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 59/151052 [00:00<04:20, 580.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "non_overlapping_dataset\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 151052/151052 [03:57<00:00, 636.10it/s]\n",
      "100%|██████████| 151052/151052 [03:34<00:00, 704.42it/s]\n",
      "  0%|          | 135/98950 [00:00<01:13, 1341.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Disorder_dataset\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 98950/98950 [01:34<00:00, 1048.31it/s]\n",
      "100%|██████████| 98950/98950 [01:24<00:00, 1174.72it/s]\n",
      "  0%|          | 192/67654 [00:00<00:35, 1901.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Species_dataset\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 67654/67654 [00:45<00:00, 1500.53it/s]\n",
      "100%|██████████| 67654/67654 [00:38<00:00, 1751.94it/s]\n",
      "  1%|          | 269/43779 [00:00<00:16, 2677.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Chemical_Drug_dataset\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 43779/43779 [00:19<00:00, 2191.35it/s]\n",
      "100%|██████████| 43779/43779 [00:16<00:00, 2700.40it/s]\n",
      "  1%|          | 336/29888 [00:00<00:08, 3356.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gene_Protein_dataset\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 29888/29888 [00:10<00:00, 2833.95it/s]\n",
      "100%|██████████| 29888/29888 [00:07<00:00, 4074.19it/s]\n",
      "  0%|          | 606/151052 [00:00<00:24, 6047.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Enzyme_dataset\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 151052/151052 [00:25<00:00, 5830.00it/s]\n",
      "100%|██████████| 151052/151052 [00:10<00:00, 14513.13it/s]\n",
      "  3%|▎         | 216/6188 [00:00<00:02, 2157.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Anatomy_dataset\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6188/6188 [00:03<00:00, 2001.50it/s]\n",
      "100%|██████████| 6188/6188 [00:02<00:00, 2392.14it/s]\n",
      "  0%|          | 241/50554 [00:00<00:20, 2405.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Biological_Process_dataset\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 50554/50554 [00:25<00:00, 1961.53it/s]\n",
      "100%|██████████| 50554/50554 [00:21<00:00, 2368.57it/s]\n",
      "  5%|▌         | 605/11398 [00:00<00:01, 6046.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Molecular_Function_dataset\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 11398/11398 [00:02<00:00, 5530.12it/s]\n",
      "100%|██████████| 11398/11398 [00:01<00:00, 11081.20it/s]\n",
      "  1%|▏         | 402/27208 [00:00<00:06, 4012.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cellular_Component_dataset\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 27208/27208 [00:08<00:00, 3192.33it/s]\n",
      "100%|██████████| 27208/27208 [00:06<00:00, 4398.19it/s]\n",
      " 16%|█▌        | 907/5637 [00:00<00:00, 9067.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pathway_dataset\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5637/5637 [00:00<00:00, 8008.20it/s]\n",
      "100%|██████████| 5637/5637 [00:00<00:00, 22869.79it/s]\n",
      "100%|██████████| 10/10 [00:00<00:00, 4028.34it/s]\n",
      "100%|██████████| 10/10 [00:00<00:00, 26852.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "microRNA_dataset\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "non_overlapping_dataset \n",
    "print('non_overlapping_dataset')\n",
    "trainidx, develidx, testidx = get_train_dev_test_indxs(len(non_overlapping_dataset))\n",
    "convert_to_IOB_format(non_overlapping_dataset,trainidx, develidx, testidx, cwd_, 'BIO/non_overlapping_dataset/' )\n",
    "convert_to_jsonl_format(non_overlapping_dataset,trainidx, develidx, testidx, cwd_, 'SpaCy/non_overlapping_dataset/' )\n",
    "\n",
    "\n",
    "# overlapping_dataset\n",
    "# trainidx, develidx, testidx = get_train_dev_test_indxs(len(overlapping_dataset))\n",
    "# convert_to_IOB_format(non_overlapping_dataset,trainidx, develidx, testidx, cwd_, 'BIO/overlapping_dataset/' )\n",
    "# convert_to_jsonl_format(non_overlapping_dataset,trainidx, develidx, testidx, cwd_, 'SpaCy/overlapping_dataset/' )\n",
    "\n",
    "# # Disorder_dataset\n",
    "# print('Disorder_dataset')\n",
    "# trainidx, develidx, testidx = get_train_dev_test_indxs(len(Disorder_dataset))\n",
    "# convert_to_IOB_format(Disorder_dataset,trainidx, develidx, testidx, cwd_, 'BIO/Disorder_dataset/' )\n",
    "# convert_to_jsonl_format(Disorder_dataset,trainidx, develidx, testidx, cwd_, 'SpaCy/Disorder_dataset/' )\n",
    "\n",
    "Species_dataset\n",
    "# print('Species_dataset')\n",
    "# trainidx, develidx, testidx = get_train_dev_test_indxs(len(Species_dataset))\n",
    "# convert_to_IOB_format(Species_dataset,trainidx, develidx, testidx, cwd_, 'BIO/Species_dataset/' )\n",
    "# convert_to_jsonl_format(Species_dataset,trainidx, develidx, testidx, cwd_, 'SpaCy/Species_dataset/' )\n",
    "\n",
    "# Chemical_Drug_dataset\n",
    "print('Chemical_Drug_dataset')\n",
    "trainidx, develidx, testidx = get_train_dev_test_indxs(len(Chemical_Drug_dataset))\n",
    "convert_to_IOB_format(Chemical_Drug_dataset,trainidx, develidx, testidx, cwd_, 'BIO/Chemical_Drug_dataset/' )\n",
    "convert_to_jsonl_format(Chemical_Drug_dataset,trainidx, develidx, testidx, cwd_, 'SpaCy/Chemical_Drug_dataset/' )\n",
    "\n",
    "# # Gene_Protein_dataset\n",
    "# print('Gene_Protein_dataset')\n",
    "# trainidx, develidx, testidx = get_train_dev_test_indxs(len(Gene_Protein_dataset))\n",
    "# convert_to_IOB_format(Gene_Protein_dataset,trainidx, develidx, testidx, cwd_, 'BIO/Gene_Protein_dataset/' )\n",
    "# convert_to_jsonl_format(Gene_Protein_dataset,trainidx, develidx, testidx, cwd_, 'SpaCy/Gene_Protein_dataset/' )\n",
    "\n",
    "# # Enzyme_dataset\n",
    "# print('Enzyme_dataset')\n",
    "# trainidx, develidx, testidx = get_train_dev_test_indxs(len(Enzyme_dataset))\n",
    "# convert_to_IOB_format(Enzyme_dataset,trainidx, develidx, testidx, cwd_, 'BIO/Enzyme_dataset/' )\n",
    "# convert_to_jsonl_format(Enzyme_dataset,trainidx, develidx, testidx, cwd_, 'SpaCy/Enzyme_dataset/' )\n",
    "\n",
    "# # Anatomy_dataset\n",
    "# print('Anatomy_dataset')\n",
    "# trainidx, develidx, testidx = get_train_dev_test_indxs(len(Anatomy_dataset))\n",
    "# convert_to_IOB_format(Anatomy_dataset,trainidx, develidx, testidx, cwd_, 'BIO/Anatomy_dataset/' )\n",
    "# convert_to_jsonl_format(Anatomy_dataset,trainidx, develidx, testidx, cwd_, 'SpaCy/Anatomy_dataset/' )\n",
    "\n",
    "# Biological_Process_dataset\n",
    "print('Biological_Process_dataset')\n",
    "trainidx, develidx, testidx = get_train_dev_test_indxs(len(Biological_Process_dataset))\n",
    "convert_to_IOB_format(Biological_Process_dataset,trainidx, develidx, testidx, cwd_, 'BIO/Biological_Process_dataset/' )\n",
    "convert_to_jsonl_format(Biological_Process_dataset,trainidx, develidx, testidx, cwd_, 'SpaCy/Biological_Process_dataset/' )\n",
    "\n",
    "# # Molecular_Function_dataset\n",
    "# print('Molecular_Function_dataset')\n",
    "# trainidx, develidx, testidx = get_train_dev_test_indxs(len(Molecular_Function_dataset))\n",
    "# convert_to_IOB_format(Molecular_Function_dataset,trainidx, develidx, testidx, cwd_, 'BIO/Molecular_Function_dataset/' )\n",
    "# convert_to_jsonl_format(Molecular_Function_dataset,trainidx, develidx, testidx, cwd_, 'SpaCy/Molecular_Function_dataset/' )\n",
    "\n",
    "# # Cellular_Component_dataset\n",
    "# print('Cellular_Component_dataset')\n",
    "# trainidx, develidx, testidx = get_train_dev_test_indxs(len(Cellular_Component_dataset))\n",
    "# convert_to_IOB_format(Cellular_Component_dataset,trainidx, develidx, testidx, cwd_, 'BIO/Cellular_Component_dataset/' )\n",
    "# convert_to_jsonl_format(Cellular_Component_dataset,trainidx, develidx, testidx, cwd_, 'SpaCy/Cellular_Component_dataset/' )\n",
    "\n",
    "# # Pathway_dataset\n",
    "# print('Pathway_dataset')\n",
    "# trainidx, develidx, testidx = get_train_dev_test_indxs(len(Pathway_dataset))\n",
    "# convert_to_IOB_format(Pathway_dataset,trainidx, develidx, testidx, cwd_, 'BIO/Pathway_dataset/' )\n",
    "# convert_to_jsonl_format(Pathway_dataset,trainidx, develidx, testidx, cwd_, 'SpaCy/Pathway_dataset/' )\n",
    "\n",
    "# # microRNA_dataset\n",
    "# print('microRNA_dataset')\n",
    "# trainidx, develidx, testidx = get_train_dev_test_indxs(len(microRNA_dataset))\n",
    "# convert_to_IOB_format(microRNA_dataset,trainidx, develidx, testidx, cwd_, 'BIO/microRNA_dataset/' )\n",
    "# convert_to_jsonl_format(microRNA_dataset,trainidx, develidx, testidx, cwd_, 'SpaCy/microRNA_dataset/' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12%|█▏        | 725/6188 [00:00<00:00, 7248.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Enzyme_dataset\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6188/6188 [00:00<00:00, 7131.38it/s]\n",
      "100%|██████████| 6188/6188 [00:00<00:00, 19277.85it/s]\n",
      "  0%|          | 224/59830 [00:00<00:26, 2234.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Anatomy_dataset\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 59830/59830 [00:37<00:00, 1600.37it/s]\n",
      "100%|██████████| 59830/59830 [00:31<00:00, 1919.05it/s]\n"
     ]
    }
   ],
   "source": [
    "# # Enzyme_dataset\n",
    "# print('Enzyme_dataset')\n",
    "# trainidx, develidx, testidx = get_train_dev_test_indxs(len(Enzyme_dataset))\n",
    "# convert_to_IOB_format(Enzyme_dataset,trainidx, develidx, testidx, cwd_, 'BIO/Enzyme_dataset/' )\n",
    "# convert_to_jsonl_format(Enzyme_dataset,trainidx, develidx, testidx, cwd_, 'SpaCy/Enzyme_dataset/' )\n",
    "\n",
    "# # Anatomy_dataset\n",
    "# print('Anatomy_dataset')\n",
    "# trainidx, develidx, testidx = get_train_dev_test_indxs(len(Anatomy_dataset))\n",
    "# convert_to_IOB_format(Anatomy_dataset,trainidx, develidx, testidx, cwd_, 'BIO/Anatomy_dataset/' )\n",
    "# convert_to_jsonl_format(Anatomy_dataset,trainidx, develidx, testidx, cwd_, 'SpaCy/Anatomy_dataset/' )\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}