Unsup-FinBERT-tone.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c7ed7fa1-39a9-4430-92bd-b5faacad168c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "addb0b0f9aa64ab5ab6faf0a70bb093a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading config.json:   0%|          | 0.00/533 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "829b732b9790469986c4fecb5e61e40c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading pytorch_model.bin:   0%|          | 0.00/419M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "27d9a5cd952948f4a358f0f86b5fa7c8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading vocab.txt:   0%|          | 0.00/221k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'label': 'Positive', 'score': 0.9999995231628418}]\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertTokenizer, BertForSequenceClassification, pipeline\n",
    "\n",
    "finbert = BertForSequenceClassification.from_pretrained('yiyanghkust/finbert-tone')\n",
    "tokenizer = BertTokenizer.from_pretrained('yiyanghkust/finbert-tone')\n",
    "nlp = pipeline(\"text-classification\", model=finbert, tokenizer=tokenizer)\n",
    "results = nlp('We expect the age of our fleet to enhance availability and reliability due to reduced downtime for repairs.')\n",
    "print(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "793297d2-88b9-4087-b553-1be7bdd2c704",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import time\n",
    "\n",
    "import nltk\n",
    "\n",
    "from tqdm.notebook import trange, tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "68416911-e4d9-4bbe-9f14-7d139dcec0fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('ERAI Dataset_with_translation/ERAI_Dataset_unsupervised_with_translation.json') as fp:\n",
    "    data = json.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6609cfc6-47d7-417e-b346-b29aaa9d9eb0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import sent_tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "cf027df3-d442-4151-950d-e53f0ce363aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "label_set = ['Neutral','Positive','Negative']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a779bfd9-9841-400e-b2f6-9637434549bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = {} # index:(label, score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ed902968-4b69-441e-80ae-32aeda278a51",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "da9dace3c8614db19524527306a5c84d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/210 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for d in tqdm(data):\n",
    "    sents = sent_tokenize(d['post_rationale_en'])\n",
    "    labs = dict.fromkeys(label_set, 0)\n",
    "    for s in sents:\n",
    "        r = nlp(s)\n",
    "        labs[r[0]['label']] += r[0]['score']\n",
    "    scores[d['index']] = labs\n",
    "    time.sleep(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "48081154-140f-40a1-8fe9-a105d67f1373",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('BuildingLexicon_code/scores/FinBERT-tone.json', 'w') as f:\n",
    "    json.dump(scores, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6cbbaf9-d0d0-4d61-ba53-8aa945b94f96",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "661f3bd4-b847-4544-a37d-e7d415564406",
   "metadata": {},
   "outputs": [],
   "source": [
    "jf_path = 'BuildingLexicon_code/scores/FinBERT-tone.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8dc533e4-1136-44e8-b75d-c8babebf3d77",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = dict(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "18ea28c5-ba31-4c76-bca9-60e1454724a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "list0 = {d:data[d][label_set[0]] for d in range(len(data))}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "8fa50087-0fd2-4f77-a102-d380661798ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "list1 = {d:data[d][label_set[1]] for d in range(len(data))}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "06c8c071-98f3-496d-b6c8-d8f7cdd13be5",
   "metadata": {},
   "outputs": [],
   "source": [
    "list2 = {d:data[d][label_set[2]] for d in range(len(data))}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "9d4e79af-15cd-4304-91cf-a766faccce34",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('BuildingLexicon_code/scores/FinBERT-tone_{}.json'.format(label_set[0]), 'w') as f:\n",
    "    json.dump(list0, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "a1a4a920-09e5-4e98-951b-5796439d4595",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('BuildingLexicon_code/scores/FinBERT-tone_{}.json'.format(label_set[1]), 'w') as f:\n",
    "    json.dump(list1, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "c3ede18c-16c3-4972-9a73-7fa8b07a4a7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('BuildingLexicon_code/scores/FinBERT-tone_{}.json'.format(label_set[2]), 'w') as f:\n",
    "    json.dump(list2, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68f5e70a-a0ee-4a4b-9055-395a646babbb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "969374dc-a6ff-4cd3-8fb4-e2b347bbd9fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "mv = max(list0.values())\n",
    "for d in list0:\n",
    "    list0[d] /= mv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "fcc1facf-4e6f-40b7-8178-11a50df9c305",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('BuildingLexicon_code/scores/Normalized/'+'FinBERT-tone_{}.json'.format(label_set[0]), 'w') as f:\n",
    "    json.dump(list0, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "5e0307f9-779c-49de-a871-b6583eac21cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "mv = max(list1.values())\n",
    "for d in list1:\n",
    "    list1[d] /= mv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "7f9649b3-cb5a-4c53-8c71-26e43e1aaa4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('BuildingLexicon_code/scores/Normalized/'+'FinBERT-tone_{}.json'.format(label_set[1]), 'w') as f:\n",
    "    json.dump(list1, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "35363705-b7ba-46a8-91e2-7f7cecad8e10",
   "metadata": {},
   "outputs": [],
   "source": [
    "mv = max(list2.values())\n",
    "for d in list2:\n",
    "    list2[d] /= mv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "d37b8907-029b-451b-a3bf-cb237eeaad01",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('BuildingLexicon_code/scores/Normalized/'+'FinBERT-tone_{}.json'.format(label_set[2]), 'w') as f:\n",
    "    json.dump(list2, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b0499eb-a724-48fc-8971-9b559b2c9301",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "206b69ec-af67-49b9-acd9-7f1c43039134",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}