![\"Open](\"https://colab.research.google.com/assets/colab-badge.svg\"){kind=icon}
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "xSDjs5OLWfzr"
+ },
+ "source": [
+ "# Installing Graph-Tool in Colab\n",
+ "\n",
+ "[https://colab.research.google.com/github/count0/colab-gt/blob/master/colab-gt.ipynb#scrollTo=ccAj89hUOqoU](https://colab.research.google.com/github/count0/colab-gt/blob/master/colab-gt.ipynb#scrollTo=ccAj89hUOqoU)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "0YHPNyQOWd5E",
+ "outputId": "508dad20-c681-45db-f4b5-bfec59ab77da"
+ },
+ "outputs": [],
+ "source": [
+ "!wget https://downloads.skewed.de/skewed-keyring/skewed-keyring_1.0_all_$(lsb_release -s -c).deb\n",
+ "!dpkg -i skewed-keyring_1.0_all_$(lsb_release -s -c).deb\n",
+ "!echo \"deb [signed-by=/usr/share/keyrings/skewed-keyring.gpg] https://downloads.skewed.de/apt $(lsb_release -s -c) main\" > /etc/apt/sources.list.d/skewed.list\n",
+ "!apt-get update\n",
+ "!apt-get install python3-graph-tool python3-matplotlib python3-cairo"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "Ux-28Se_W1oY",
+ "outputId": "6f082e93-4650-4c37-bf42-d34c33c9ae89"
+ },
+ "outputs": [],
+ "source": [
+ "# Colab uses a Python install that deviates from the system's! Bad colab! We need some workarounds.\n",
+ "!apt purge python3-cairo\n",
+ "!apt install libcairo2-dev pkg-config python3-dev\n",
+ "!pip install --force-reinstall pycairo\n",
+ "!pip install zstandard"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "-aXZPJcJXE9g",
+ "outputId": "012d9256-8ff6-4b60-d4d1-78ce208e5848"
+ },
+ "outputs": [],
+ "source": [
+ "# Copying topSBM to Colab\n",
+ "!git clone https://github.com/edugalt/hSBM_Topicmodel.git\n",
+ "!cp hSBM_Topicmodel/data/corpus.txt .\n",
+ "!cp hSBM_Topicmodel/data/titles.txt .\n",
+ "!cp hSBM_Topicmodel/sbmtm.py ."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "e17klBBOWZg5"
+ },
+ "source": [
+ "# TopSBM: Topic Modeling with Stochastic Block Models\n",
+ "\n",
+ "A basic tutorial.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "TekTGcwpWZg6",
+ "outputId": "ee17588e-690c-4ffa-8370-2968f33b2a12"
+ },
+ "outputs": [],
+ "source": [
+ "%load_ext autoreload\n",
+ "%autoreload 2\n",
+ "\n",
+ "import os\n",
+ "import pylab as plt\n",
+ "%matplotlib inline\n",
+ "\n",
+ "from sbmtm import sbmtm\n",
+ "import graph_tool.all as gt"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "K1wwZj0MWZg6"
+ },
+ "source": [
+ "# Setup: Load a corpus\n",
+ "\n",
+ "1) We have a list of documents, each document contains a list of words.\n",
+ "\n",
+ "2) We have a list of document titles (optional)\n",
+ "\n",
+ "The example corpus consists of 63 articles from Wikipedia taken from 3 different categories (Experimental Physics, Chemical Physics, and Computational Biology).\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {
+ "id": "pLy1tkXFWZg7"
+ },
+ "outputs": [],
+ "source": [
+ "path_data = ''\n",
+ "\n",
+ "## texts\n",
+ "fname_data = 'corpus.txt'\n",
+ "filename = os.path.join(path_data,fname_data)\n",
+ "\n",
+ "with open(filename,'r', encoding = 'utf8') as f:\n",
+ " x = f.readlines()\n",
+ "texts = [h.split() for h in x]\n",
+ "\n",
+ "## titles\n",
+ "fname_data = 'titles.txt'\n",
+ "filename = os.path.join(path_data,fname_data)\n",
+ "\n",
+ "with open(filename,'r', encoding = 'utf8') as f:\n",
+ " x = f.readlines()\n",
+ "titles = [h.split()[0] for h in x]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "P4jF8B4MWZg7",
+ "outputId": "0d965b71-1686-4373-bbdd-7cb623bbd5da"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Nuclear_Overhauser_effect\n",
+ "['the', 'nuclear', 'overhauser', 'effect', 'noe', 'is', 'the', 'transfer', 'of', 'nuclear', 'spin', 'polarization', 'from', 'one', 'nuclear', 'spin', 'population', 'to', 'another', 'via']\n"
+ ]
+ }
+ ],
+ "source": [
+ "i_doc = 0\n",
+ "print(titles[0])\n",
+ "print(texts[i_doc][:20])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "pvx2sRODWZg7"
+ },
+ "source": [
+ "# Fitting the model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "eeikN30hWZg8",
+ "outputId": "b6120d43-90bd-4356-d7ad-f2576c17c043"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "A basic tutorial.
- -%load_ext autoreload
-%autoreload 2
-
-import os
-import pylab as plt
-%matplotlib inline
-
-from sbmtm import sbmtm
-import graph_tool.all as gt
-1) We have a list of documents, each document contains a list of words.
-2) We have a list of document titles (optional)
-The example corpus consists of 63 articles from Wikipedia taken from 3 different categories (Experimental Physics, Chemical Physics, and Computational Biology).
- -path_data = ''
-
-## texts
-fname_data = 'corpus.txt'
-filename = os.path.join(path_data,fname_data)
-
-with open(filename,'r', encoding = 'utf8') as f:
- x = f.readlines()
-texts = [h.split() for h in x]
-
-## titles
-fname_data = 'titles.txt'
-filename = os.path.join(path_data,fname_data)
-
-with open(filename,'r', encoding = 'utf8') as f:
- x = f.readlines()
-titles = [h.split()[0] for h in x]
-i_doc = 0
-print(titles[0])
-print(texts[i_doc][:10])
-Nuclear_Overhauser_effect -['the', 'nuclear', 'overhauser', 'effect', 'noe', 'is', 'the', 'transfer', 'of', 'nuclear'] --
## we create an instance of the sbmtm-class
-model = sbmtm()
-
-## we have to create the word-document network from the corpus
-model.make_graph(texts,documents=titles)
-
-## we can also skip the previous step by saving/loading a graph
-# model.save_graph(filename = 'graph.xml.gz')
-# model.load_graph(filename = 'graph.xml.gz')
-
-## fit the model
-gt.seed_rng(32) ## seed for graph-tool's random number generator --> same results
-model.fit()
-<NestedBlockState object, with base <BlockState object with 3193 blocks (84 nonempty), degree-corrected, for graph <Graph object, undirected, with 3203 vertices and 13050 edges, 2 internal vertex properties, 1 internal edge property, at 0x7fa4cc60c580>, at 0x7fa4113619a0>, and 5 levels of sizes [(3203, 84), (84, 22), (22, 6), (6, 2), (2, 1)] at 0x7fa4113618b0> --
The output shows the (hierarchical) community structure in the word-document network as inferred by the stochastic block model:
-The result is a grouping of nodes into groups on multiple levels in the hierarchy:
-model.plot(nedges=10000)
-
-For each word-group on a given level in the hierarchy, we retrieve the $n$ most common words in each group -- these are the topics!
- -model.topics(l=1,n=20)
-{0: [('electron', 0.02750225428313796),
- ('x', 0.023444544634806132),
- ('molecule', 0.02254283137962128),
- ('spin', 0.021190261496844005),
- ('surface', 0.020739404869251576),
- ('atoms', 0.020739404869251576),
- ('magnetic', 0.01848512173128945),
- ('frequency', 0.017132551848512173),
- ('because', 0.016681695220919748),
- ('electrons', 0.015779981965734897),
- ('normal', 0.015779981965734897),
- ('ray', 0.014878268710550046),
- ('nuclear', 0.013074842200180343),
- ('polarization', 0.012623985572587917),
- ('observed', 0.01127141568981064),
- ('intensity', 0.010820559062218215),
- ('sample', 0.010820559062218215),
- ('vibration', 0.010369702434625788),
- ('rotational', 0.009918845807033363),
- ('c', 0.009918845807033363)],
- 1: [('may', 0.048538011695906436),
- ('structure', 0.0456140350877193),
- ('theory', 0.027485380116959064),
- ('uncertainty', 0.027485380116959064),
- ('experimental', 0.02456140350877193),
- ('experiments', 0.023391812865497075),
- ('science', 0.019883040935672516),
- ('mechanics', 0.01929824561403509),
- ('part', 0.01871345029239766),
- ('molecules', 0.01695906432748538),
- ('classical', 0.01695906432748538),
- ('approximation', 0.014035087719298246),
- ('parallel', 0.012280701754385965),
- ('measured', 0.012280701754385965),
- ('measurement', 0.011111111111111112),
- ('standard', 0.010526315789473684),
- ('important', 0.009941520467836258),
- ('larger', 0.009941520467836258),
- ('we', 0.00935672514619883),
- ('effective', 0.00935672514619883)],
- 2: [('folding', 0.0889759373251259),
- ('protein', 0.07778399552322328),
- ('home', 0.0710688304420817),
- ('computational', 0.04308897593732513),
- ('bioinformatics', 0.0408505875769446),
- ('research', 0.03693340794627868),
- ('biology', 0.028539451594851706),
- ('proteins', 0.02574146614437605),
- ('project', 0.02294348069390039),
- ('simulations', 0.02182428651371013),
- ('new', 0.01846670397313934),
- ('computing', 0.01846670397313934),
- ('client', 0.01846670397313934),
- ('development', 0.014549524342473419),
- ('lab', 0.013989927252378288),
- ('disease', 0.013989927252378288),
- ('pande', 0.013989927252378288),
- ('public', 0.011751538891997761),
- ('user', 0.01119194180190263),
- ('clients', 0.010632344711807499)],
- 3: [('formula', 0.2597560975609756),
- ('scattering', 0.03414634146341464),
- ('polarizability', 0.03048780487804878),
- ('case', 0.02926829268292683),
- ('nucleon', 0.025609756097560974),
- ('polarizabilities', 0.023170731707317073),
- ('defined', 0.018292682926829267),
- ('pattern', 0.01707317073170732),
- ('compton', 0.01707317073170732),
- ('neutron', 0.014634146341463415),
- ('empirical', 0.013414634146341463),
- ('sum', 0.013414634146341463),
- ('constituent', 0.012195121951219513),
- ('photon', 0.012195121951219513),
- ('ratio', 0.012195121951219513),
- ('equations', 0.012195121951219513),
- ('meson', 0.012195121951219513),
- ('flow', 0.01097560975609756),
- ('rule', 0.01097560975609756),
- ('quark', 0.01097560975609756)],
- 4: [('physics', 0.09055118110236221),
- ('effect', 0.04461942257217848),
- ('physical', 0.028871391076115485),
- ('pauli', 0.027559055118110236),
- ('early', 0.024934383202099737),
- ('nature', 0.023622047244094488),
- ('radiation', 0.02230971128608924),
- ('theoretical', 0.01968503937007874),
- ('beams', 0.01837270341207349),
- ('chemistry', 0.01837270341207349),
- ('liquid', 0.01706036745406824),
- ('mechanical', 0.015748031496062992),
- ('hydrometer', 0.014435695538057743),
- ('phenomenon', 0.013123359580052493),
- ('haas', 0.013123359580052493),
- ('physicist', 0.013123359580052493),
- ('proposed', 0.013123359580052493),
- ('universe', 0.011811023622047244),
- ('century', 0.011811023622047244),
- ('equilibrium', 0.010498687664041995)],
- 5: [('law', 0.05757575757575758),
- ('momentum', 0.05656565656565657),
- ('force', 0.048484848484848485),
- ('newton', 0.043434343434343436),
- ('motion', 0.04242424242424243),
- ('body', 0.04141414141414142),
- ('applied', 0.03838383838383838),
- ('laws', 0.03636363636363636),
- ('second', 0.031313131313131314),
- ('he', 0.028282828282828285),
- ('angular', 0.023232323232323233),
- ('fields', 0.022222222222222223),
- ('like', 0.022222222222222223),
- ('vector', 0.01717171717171717),
- ('forces', 0.01616161616161616),
- ('conservation', 0.01616161616161616),
- ('objects', 0.012121212121212121),
- ('acceleration', 0.012121212121212121),
- ('third', 0.011111111111111112),
- ('bodies', 0.011111111111111112)],
- 6: [('s', 0.041019955654102),
- ('was', 0.03436807095343681),
- ('data', 0.019290465631929047),
- ('its', 0.01796008869179601),
- ('first', 0.017738359201773836),
- ('time', 0.014190687361419069),
- ('scientific', 0.011086474501108648),
- ('sequence', 0.010421286031042129),
- ('computer', 0.009977827050997782),
- ('information', 0.009534368070953437),
- ('software', 0.009312638580931265),
- ('based', 0.00909090909090909),
- ('over', 0.008869179600886918),
- ('many', 0.008869179600886918),
- ('genome', 0.008647450110864745),
- ('methods', 0.008425720620842573),
- ('university', 0.008203991130820399),
- ('available', 0.00753880266075388),
- ('function', 0.00753880266075388),
- ('biological', 0.00753880266075388)],
- 7: [('assembly', 0.046620046620046623),
- ('de', 0.04428904428904429),
- ('transcriptome', 0.037296037296037296),
- ('transcripts', 0.027972027972027972),
- ('networks', 0.027972027972027972),
- ('low', 0.02564102564102564),
- ('contigs', 0.02097902097902098),
- ('graphs', 0.018648018648018648),
- ('read', 0.016317016317016316),
- ('assembled', 0.016317016317016316),
- ('neural', 0.016317016317016316),
- ('sepp', 0.016317016317016316),
- ('hochreiter', 0.016317016317016316),
- ('farms', 0.016317016317016316),
- ('reads', 0.016317016317016316),
- ('isoforms', 0.013986013986013986),
- ('go', 0.013986013986013986),
- ('cdna', 0.011655011655011656),
- ('novo', 0.011655011655011656),
- ('psvm', 0.011655011655011656)],
- 8: [('if', 0.03254437869822485),
- ('mass', 0.029585798816568046),
- ('quantum', 0.028106508875739646),
- ('source', 0.027613412228796843),
- ('point', 0.026134122287968443),
- ('state', 0.02465483234714004),
- ('will', 0.023175542406311637),
- ('linear', 0.021696252465483234),
- ('due', 0.01972386587771203),
- ('light', 0.019230769230769232),
- ('wave', 0.015285996055226824),
- ('sources', 0.014792899408284023),
- ('particle', 0.014299802761341223),
- ('change', 0.013806706114398421),
- ('air', 0.013313609467455622),
- ('waves', 0.01282051282051282),
- ('water', 0.01232741617357002),
- ('following', 0.01232741617357002),
- ('particles', 0.011834319526627219),
- ('noise', 0.011834319526627219)],
- 9: [('atomic', 0.02195121951219512),
- ('velocity', 0.01951219512195122),
- ('detector', 0.018902439024390243),
- ('dimensional', 0.016463414634146342),
- ('center', 0.016463414634146342),
- ('imaging', 0.015853658536585366),
- ('distribution', 0.01524390243902439),
- ('real', 0.01524390243902439),
- ('product', 0.01402439024390244),
- ('therefore', 0.01402439024390244),
- ('constant', 0.013414634146341463),
- ('reference', 0.012804878048780487),
- ('frame', 0.012195121951219513),
- ('ion', 0.012195121951219513),
- ('laser', 0.012195121951219513),
- ('above', 0.012195121951219513),
- ('space', 0.011585365853658536),
- ('o', 0.01097560975609756),
- ('respect', 0.01097560975609756),
- ('products', 0.01097560975609756)],
- 10: [('field', 0.18248175182481752),
- ('energy', 0.12408759124087591),
- ('electric', 0.09854014598540146),
- ('beam', 0.06934306569343066),
- ('technique', 0.06204379562043796),
- ('atom', 0.05474452554744526),
- ('chemical', 0.051094890510948905),
- ('direction', 0.041970802919708027),
- ('charge', 0.032846715328467155),
- ('measure', 0.032846715328467155),
- ('relative', 0.032846715328467155),
- ('dipole', 0.029197080291970802),
- ('short', 0.02737226277372263),
- ('induced', 0.021897810218978103),
- ('ground', 0.020072992700729927),
- ('provided', 0.01824817518248175),
- ('axis', 0.016423357664233577),
- ('deflection', 0.014598540145985401),
- ('element', 0.012773722627737226),
- ('should', 0.012773722627737226)],
- 11: [('the', 0.23024895668441503),
- ('of', 0.12994675492876673),
- ('a', 0.07540653331414592),
- ('to', 0.07310404374730177),
- ('in', 0.07044179018563822),
- ('as', 0.028709166786587997),
- ('that', 0.02324075406533314),
- ('by', 0.02057850050366959),
- ('with', 0.017628435746150525),
- ('an', 0.017412577349258888),
- ('this', 0.01604547416894517),
- ('can', 0.014894229385523096),
- ('or', 0.01388689020002878),
- ('from', 0.013814937401064901),
- ('it', 0.013527126205209383),
- ('at', 0.011512447834220752),
- ('not', 0.008562383076701683),
- ('has', 0.008418477478773924),
- ('such', 0.0076989494891351274),
- ('these', 0.007483091092243488)],
- 12: [('and', 0.13414634146341464),
- ('is', 0.09696317551410809),
- ('for', 0.05117168818747011),
- ('are', 0.036944045911047343),
- ('be', 0.03311812529890005),
- ('on', 0.02965088474414156),
- ('which', 0.024509803921568627),
- ('used', 0.01446676231468197),
- ('two', 0.010760401721664276),
- ('when', 0.009445241511238642),
- ('been', 0.00824964131994261),
- ('analysis', 0.007771401243424199),
- ('where', 0.007054041128646581),
- ('their', 0.0068149210903873745),
- ('about', 0.006217120994739359),
- ('there', 0.00585844093735055),
- ('systems', 0.005738880918220947),
- ('using', 0.005738880918220947),
- ('work', 0.005499760879961741),
- ('model', 0.005380200860832138)]}
-Which topics contribute to each document?
- -## select a document (by its index)
-i_doc = 0
-print(model.documents[i_doc])
-## get a list of tuples (topic-index, probability)
-model.topicdist(i_doc,l=1)
-Nuclear_Overhauser_effect --
[(0, 0.2692307692307692), - (1, 0.013986013986013986), - (2, 0.0034965034965034965), - (3, 0.006993006993006993), - (4, 0.038461538461538464), - (5, 0.0034965034965034965), - (6, 0.08041958041958042), - (7, 0.0034965034965034965), - (8, 0.013986013986013986), - (9, 0.04195804195804196), - (10, 0.01048951048951049), - (11, 0.36713286713286714), - (12, 0.14685314685314685)]-
-The stochastic block models clusters the documents into groups. -We do not need to run an additional clustering to obtain this grouping.
- -model.clusters(l=1,n=5)
-{0: [('Philosophical_interpretation_of_classical_physics', 1.0),
- ('Uncertainty', 1.0),
- ('Law_of_Maximum', 1.0)],
- 1: [('De_novo_transcriptome_assembly', 1.0), ('Sepp_Hochreiter', 1.0)],
- 2: [('BioUML', 1.0), ('Folding@home', 1.0), ('Bioinformatics', 1.0)],
- 3: [('Empirical_formula', 1.0),
- ('Ziff-Gulari-Barshad_model', 1.0),
- ('Fuel_mass_fraction', 1.0),
- ("Euler's_laws_of_motion", 1.0),
- ("Newton's_laws_of_motion", 1.0)],
- 4: [('Nuclear_Overhauser_effect', 1.0),
- ('Molecular_vibration', 1.0),
- ('Particle-induced_X-ray_emission', 1.0),
- ('X-ray_crystal_truncation_rod', 1.0),
- ('X-ray_standing_waves', 1.0)],
- 5: [('Foldit', 1.0),
- ('Louis_and_Beatrice_Laufer_Center_for_Physical_and_Quantitative_Biology',
- 1.0),
- ('Computational_biology', 1.0),
- ('K-mer', 1.0),
- ('Journal_of_Computational_Biology', 1.0)],
- 6: [('Electrostatic_deflection_(structural_element)', 1.0),
- ('Molecular_beam', 1.0),
- ('Experimental_physics', 1.0),
- ('Pauli_effect', 1.0),
- ('Einstein–de_Haas_effect', 1.0)],
- 7: [('Wave_tank', 1.0),
- ('Reactive_empirical_bond_order', 1.0),
- ('Ripple_tank', 1.0),
- ('Faraday_cup_electrometer', 1.0),
- ('Line_source', 1.0)],
- 8: [('Polarizability', 1.0),
- ('Effective_field_theory', 1.0),
- ('RRKM_theory', 1.0),
- ('Rotating_wave_approximation', 1.0),
- ('Quantum_oscillations_(experimental_technique)', 1.0)]}
-Application -- Finding similar articles:
-For a query-article, we return all articles from the same group
- -## select a document (index)
-i_doc = 2
-print(i_doc,model.documents[i_doc])
-## find all articles from the same group
-## print: (doc-index, doc-title)
-model.clusters_query(i_doc,l=1,)
-2 Rovibrational_coupling --
[(0, 'Nuclear_Overhauser_effect'), - (5, 'Rotational_transition'), - (6, 'Dynamic_nuclear_polarisation'), - (7, 'Knight_shift'), - (12, 'Molecular_vibration'), - (15, 'Magic_angle_(EELS)'), - (17, 'Photofragment-ion_imaging'), - (27, 'Particle-induced_X-ray_emission'), - (29, 'Complementary_experiments'), - (33, 'X-ray_crystal_truncation_rod'), - (36, 'X-ray_standing_waves'), - (40, 'Fragment_separator')]-
In the stochastic block model, word (-nodes) and document (-nodes) are clustered into different groups.
-The group membership can be represented by the conditional probability $P(\text{group}\, |\, \text{node})$. Since words and documents belong to different groups (the word-document network is bipartite) we can show separately:
-p_td_d,p_tw_w = model.group_membership(l=1)
-
-plt.figure(figsize=(15,4))
-plt.subplot(121)
-plt.imshow(p_td_d,origin='lower',aspect='auto',interpolation='none')
-plt.title(r'Document group membership $P(bd | d)$')
-plt.xlabel('Document d (index)')
-plt.ylabel('Document group, bd')
-plt.colorbar()
-
-plt.subplot(122)
-plt.imshow(p_tw_w,origin='lower',aspect='auto',interpolation='none')
-plt.title(r'Word group membership $P(bw | w)$')
-plt.xlabel('Word w (index)')
-plt.ylabel('Word group, bw')
-plt.colorbar()
-<matplotlib.colorbar.Colorbar at 0x7fa410daadc0>-
-
-
-