CORD-19-MAG-test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Waiting for a Spark session to start..."
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "source": "user"
    },
    {
     "data": {
      "text/plain": [
       "sqlContext = org.apache.spark.sql.SQLContext@47c0373b\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "source": "user"
    },
    {
     "data": {
      "text/plain": [
       "warning: there was one deprecation warning; re-run with -deprecation for details\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "source": "user"
    },
    {
     "data": {
      "text/plain": [
       "Some(http://iuni2:4040)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import org.apache.spark.sql.{SQLContext, SaveMode}\n",
    "import org.apache.spark.sql.types.{StructType, StructField, StringType, DoubleType};\n",
    "import com.databricks.spark.xml._\n",
    "\n",
    "val sqlContext = new SQLContext(sc)\n",
    "import sqlContext.implicits._\n",
    "sqlContext.sql(\"set spark.sql.caseSensitive=true\") \n",
    "spark.sparkContext.uiWebUrl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Name: org.apache.spark.SparkException\n",
       "Message: Job 0 cancelled as part of cancellation of all jobs\n",
       "StackTrace:   at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1890)\n",
       "  at org.apache.spark.scheduler.DAGScheduler.handleJobCancellation(DAGScheduler.scala:1825)\n",
       "  at org.apache.spark.scheduler.DAGScheduler$$anonfun$doCancelAllJobs$1.apply$mcVI$sp(DAGScheduler.scala:833)\n",
       "  at org.apache.spark.scheduler.DAGScheduler$$anonfun$doCancelAllJobs$1.apply(DAGScheduler.scala:833)\n",
       "  at org.apache.spark.scheduler.DAGScheduler$$anonfun$doCancelAllJobs$1.apply(DAGScheduler.scala:833)\n",
       "  at scala.collection.mutable.HashSet.foreach(HashSet.scala:78)\n",
       "  at org.apache.spark.scheduler.DAGScheduler.doCancelAllJobs(DAGScheduler.scala:833)\n",
       "  at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2083)\n",
       "  at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2060)\n",
       "  at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2049)\n",
       "  at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)\n",
       "  at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:740)\n",
       "  at org.apache.spark.SparkContext.runJob(SparkContext.scala:2081)\n",
       "  at org.apache.spark.SparkContext.runJob(SparkContext.scala:2178)\n",
       "  at org.apache.spark.rdd.RDD$$anonfun$fold$1.apply(RDD.scala:1098)\n",
       "  at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)\n",
       "  at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)\n",
       "  at org.apache.spark.rdd.RDD.withScope(RDD.scala:363)\n",
       "  at org.apache.spark.rdd.RDD.fold(RDD.scala:1092)\n",
       "  at com.databricks.spark.xml.util.InferSchema$.infer(InferSchema.scala:111)\n",
       "  at com.databricks.spark.xml.XmlRelation$$anonfun$1.apply(XmlRelation.scala:42)\n",
       "  at com.databricks.spark.xml.XmlRelation$$anonfun$1.apply(XmlRelation.scala:42)\n",
       "  at scala.Option.getOrElse(Option.scala:121)\n",
       "  at com.databricks.spark.xml.XmlRelation.<init>(XmlRelation.scala:41)\n",
       "  at com.databricks.spark.xml.XmlRelation$.apply(XmlRelation.scala:29)\n",
       "  at com.databricks.spark.xml.DefaultSource.createRelation(DefaultSource.scala:73)\n",
       "  at com.databricks.spark.xml.DefaultSource.createRelation(DefaultSource.scala:51)\n",
       "  at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:317)\n",
       "  at org.apache.spark.sql.DataFrameReader.loadV1Source(DataFrameReader.scala:223)\n",
       "  at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:211)\n",
       "  at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:178)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "//val sqlContext = new SQLContext(sc)\n",
    "val df = sqlContext.read\n",
    "  .format(\"com.databricks.spark.xml\")\n",
    "  .option(\"rowTag\", \"REC\")\n",
    "  //.schema(customSchema)\n",
    "  .load(\"/dataRCSC/*.xml\")\n",
    "\n",
    "df.show(2)\n",
    "df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Name: Unknown Error\n",
       "Message: lastException: Throwable = null\n",
       "<console>:35: error: not found: value df\n",
       "       val WoS2 = df.select(\"UID\",\"dynamic_data.cluster_related.identifiers.identifier\",\"static_data.summary.doctypes.doctype\",\n",
       "                  ^\n",
       "\n",
       "StackTrace: "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val WoS2 = df.select(\"UID\",\"dynamic_data.cluster_related.identifiers.identifier\",\"static_data.summary.doctypes.doctype\",\n",
    "                      \"static_data.summary.names.name\",\"static_data.summary.titles\",\"static_data.summary.conferences.conference\",\n",
    "                      \"static_data.fullrecord_metadata.addresses.address_name\", \"static_data.fullrecord_metadata.reprint_addresses\",\n",
    "                      \"static_data.fullrecord_metadata.abstracts.abstract.abstract_text\",\n",
    "                      \"static_data.fullrecord_metadata.keywords.keyword\",\"static_data.fullrecord_metadata.references.reference\",\n",
    "                      \"static_data.contributors.contributor\",\"static_data.summary.EWUID\",\n",
    "                      \"static_data.summary.pub_info\",\"static_data.summary.publishers\",\"static_data.fullrecord_metadata.fund_ack\")\n",
    "WoS2.printSchema"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "WoSref = [citing: string, refList: array<string> ... 1 more field]\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "source": "user"
    },
    {
     "data": {
      "text/plain": [
       "[citing: string, refList: array<string> ... 1 more field]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import scala.util.Try\n",
    "import org.apache.spark.sql.functions._\n",
    "import org.apache.spark.sql.{Row, SQLContext}\n",
    "import org.apache.spark.sql.functions.countDistinct\n",
    "import org.apache.spark.sql.SaveMode\n",
    "\n",
    "//WoS2.write.mode(SaveMode.Overwrite).json(\"/dataRCSC/WoS_json\")\n",
    "val WoSref = WoS2.select(\"UID\",\"reference.uid\").toDF(\"citing\",\"refList\").withColumn(\"cited\", explode(col(\"refList\")))\n",
    "WoSref.select(\"citing\",\"cited\").orderBy($\"cited\".desc).coalesce(1).write.mode(SaveMode.Overwrite)\n",
    "    .format(\"com.databricks.spark.csv\")\n",
    "    .option(\"header\", \"true\").option(\"compression\", \"gzip\")\n",
    "    .save(\"/dataRCSC/WoS_json/citeEdges.csv/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "WoSref = [citing: string, refList: array<string> ... 1 more field]\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "source": "user"
    },
    {
     "data": {
      "text/plain": [
       "[citing: string, refList: array<string> ... 1 more field]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val WoSref = WoS2.select(\"UID\",\"reference.uid\").toDF(\"citing\",\"refList\").withColumn(\"cited\", explode(col(\"refList\")))\n",
    "WoSref.select(\"citing\",\"cited\").coalesce(1).write.mode(SaveMode.Overwrite)\n",
    "    .format(\"com.databricks.spark.csv\")\n",
    "    .option(\"header\", \"true\").option(\"compression\", \"gzip\")\n",
    "    .save(\"/dataRCSC/WoS_json/refEdges.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- UID: string (nullable = true)\n",
      " |-- titles: struct (nullable = true)\n",
      " |    |-- _count: long (nullable = true)\n",
      " |    |-- title: array (nullable = true)\n",
      " |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |-- _VALUE: string (nullable = true)\n",
      " |    |    |    |-- _translated: string (nullable = true)\n",
      " |    |    |    |-- _type: string (nullable = true)\n",
      " |-- category_info: struct (nullable = true)\n",
      " |    |-- headings: struct (nullable = true)\n",
      " |    |    |-- _count: long (nullable = true)\n",
      " |    |    |-- heading: array (nullable = true)\n",
      " |    |    |    |-- element: string (containsNull = true)\n",
      " |    |-- subheadings: struct (nullable = true)\n",
      " |    |    |-- _count: long (nullable = true)\n",
      " |    |    |-- subheading: array (nullable = true)\n",
      " |    |    |    |-- element: string (containsNull = true)\n",
      " |    |-- subjects: struct (nullable = true)\n",
      " |    |    |-- _count: long (nullable = true)\n",
      " |    |    |-- subject: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- _VALUE: string (nullable = true)\n",
      " |    |    |    |    |-- _ascatype: string (nullable = true)\n",
      " |-- name: array (nullable = true)\n",
      " |    |-- element: struct (containsNull = true)\n",
      " |    |    |-- _addr_no: string (nullable = true)\n",
      " |    |    |-- _orcid_id_tr: string (nullable = true)\n",
      " |    |    |-- _r_id_tr: string (nullable = true)\n",
      " |    |    |-- _reprint: string (nullable = true)\n",
      " |    |    |-- _role: string (nullable = true)\n",
      " |    |    |-- _seq_no: long (nullable = true)\n",
      " |    |    |-- display_name: string (nullable = true)\n",
      " |    |    |-- email_addr: string (nullable = true)\n",
      " |    |    |-- first_name: string (nullable = true)\n",
      " |    |    |-- full_name: string (nullable = true)\n",
      " |    |    |-- last_name: string (nullable = true)\n",
      " |    |    |-- suffix: string (nullable = true)\n",
      " |    |    |-- wos_standard: string (nullable = true)\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "WoS = [UID: string, _r_id_disclaimer: string ... 13 more fields]\n",
       "WoS2 = [UID: string, titles: struct<_count: bigint, title: array<struct<_VALUE:string,_translated:string,_type:string>>> ... 2 more fields]\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "source": "user"
    },
    {
     "data": {
      "text/plain": [
       "[UID: string, titles: struct<_count: bigint, title: array<struct<_VALUE:string,_translated:string,_type:string>>> ... 2 more fields]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val WoS = spark.read.format(\"parquet\").load(\"/WoSraw/parquet/*.parquet\")\n",
    "\n",
    "val WoS2 = WoS.select(\"UID\",\"static_data.summary.titles\",\"static_data.fullrecord_metadata.category_info\",\"static_data.summary.names.name\")\n",
    "WoS2.printSchema"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------+----------+----------+\n",
      "|  firstCombo|comboCount|paperCount|\n",
      "+------------+----------+----------+\n",
      "|           K|   3073292|   2611856|\n",
      "|          MM|    240812|    238666|\n",
      "|          LT|     45142|     45037|\n",
      "|          DZ|     13835|     13791|\n",
      "|  Michael C.|     18552|     18495|\n",
      "|    Monalisa|       578|       578|\n",
      "|        Soni|       281|       281|\n",
      "|Douglas Cole|         8|         8|\n",
      "|       E. S.|     21086|     20940|\n",
      "|       D. E.|     20047|     19704|\n",
      "|    W. Allen|       183|       183|\n",
      "|      Helmut|     19644|     19491|\n",
      "|    Francois|     56233|     54893|\n",
      "|        Ritu|      4079|      4068|\n",
      "|    Yasunari|      2169|      2168|\n",
      "|    Noriyuki|     10453|     10415|\n",
      "|    Julianne|      1783|      1783|\n",
      "|      Hideki|     30011|     29517|\n",
      "|    Byung J.|        42|        42|\n",
      "| Jules P. P.|        74|        74|\n",
      "+------------+----------+----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "WoS3 = [UID: string, first_name: array<string> ... 1 more field]\n",
       "firstCount = [firstCombo: string, comboCount: bigint ... 1 more field]\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "source": "user"
    },
    {
     "data": {
      "text/plain": [
       "[firstCombo: string, comboCount: bigint ... 1 more field]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val WoS3 = WoS2.select(\"UID\",\"name.first_name\").withColumn(\"firstCombo\", explode(col(\"first_name\")))\n",
    "val firstCount = WoS3.groupBy(\"firstCombo\").agg(count(\"*\") as \"comboCount\", countDistinct(\"UID\") as \"paperCount\")\n",
    "firstCount.show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "firstCount.coalesce(1).write.mode(SaveMode.Overwrite)\n",
    "    .format(\"com.databricks.spark.csv\")\n",
    "    .option(\"header\", \"true\")\n",
    "    .option(\"quote\", \"\\\"\")\n",
    "    .save(\"wasbs://mag-2020-09-01@iunimag.blob.core.windows.net/2020-08-03/nameCount.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-------------+-----------+--------------+----+----------+-------------+--------+---------+------+\n",
      "|summary|affiliationID|displayName|normalizedName|wiki|paperTotal|citationTotal|Latitude|Longitude|GRIDid|\n",
      "+-------+-------------+-----------+--------------+----+----------+-------------+--------+---------+------+\n",
      "|  count|         9210|       9210|          9210|9119|      9210|         9210|    8984|     9030|  7856|\n",
      "+-------+-------------+-----------+--------------+----+----------+-------------+--------+---------+------+\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "session = org.apache.spark.sql.SparkSession@255044a5\n",
       "azureFile = /azureBlob.txt MapPartitionsRDD[59] at textFile at <console>:58\n",
       "sdf = [affiliationID: string, displayName: string ... 7 more fields]\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "source": "user"
    },
    {
     "data": {
      "text/plain": [
       "[affiliationID: string, displayName: string ... 7 more fields]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import scala.util.Try\n",
    "import org.apache.spark.sql.functions._\n",
    "import org.apache.spark.sql.{Row, SQLContext}\n",
    "import org.apache.spark.sql.functions.countDistinct\n",
    "import org.apache.spark.sql.SaveMode\n",
    "\n",
    "val session = SparkSession.builder().getOrCreate()\n",
    "\n",
    "val azureFile = sc.textFile(\"/azureBlob.txt\")\n",
    "session.conf.set(\n",
    "    \"fs.azure.account.key.iunimag.blob.core.windows.net\",\n",
    "     azureFile.collect().mkString\n",
    ")\n",
    "val sdf = spark.read.format(\"csv\")\n",
    "    .option(\"header\", \"true\")\n",
    "    .option(\"quote\", \"\\\"\")\n",
    "    .option(\"escape\", \"\\\"\")\n",
    "    //.option(\"inferSchema\",\"true\")\n",
    "    //.load(\"/dataRCSC/metaLitCovid-09-13.csv\")\n",
    "    .load(\"wasbs://mag-2020-09-01@iunimag.blob.core.windows.net/2020-08-03/MAGInstTable.csv\")\n",
    "sdf.describe().filter($\"summary\" === \"count\").show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "//sdf.printSchema\n",
    "//val magAffiliation2 = magAffiliation.select(\"_c0\",\"_c2\",\"_c3\",\"_c4\",\"_c6\",\"_c7\",\"_c8\",\"_c9\",\"_c10\",\"_c11\",\"_c12\")\n",
    "//.toDF(\"AffiliationId2\",\"normalizedName\",\"displayName\",\"GRIDid\",\"wiki\",\"paperTotal\",\"paperFamilyTotal\",\"citationTotal\", \"Iso3166Code\", \"Latitude\", \"Longitude\")\n",
    "//val sdf2 = sdf.drop(\"displayName\",\"normalizedName\",\"wiki\",\"paperTotal\",\"citationTotal\",\"Latitude\",\"Longitude\",\"GRIDid\").join(magAffiliation2, sdf(\"affiliationID\")===magAffiliation2(\"AffiliationId2\"), \"left\").cache()\n",
    "sdf2.drop(\"AffiliationId2\").coalesce(1).write.mode(SaveMode.Overwrite)\n",
    "    .format(\"com.databricks.spark.csv\")\n",
    "    .option(\"header\", \"true\")\n",
    "    .option(\"quote\", \"\\\"\")\n",
    "    .save(\"wasbs://mag-2020-09-01@iunimag.blob.core.windows.net/2020-08-03/MAGInstTableUpdate.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Name: org.apache.spark.SparkException\n",
       "Message: Job 4 cancelled as part of cancellation of all jobs\n",
       "StackTrace:   at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1890)\n",
       "  at org.apache.spark.scheduler.DAGScheduler.handleJobCancellation(DAGScheduler.scala:1825)\n",
       "  at org.apache.spark.scheduler.DAGScheduler$$anonfun$doCancelAllJobs$1.apply$mcVI$sp(DAGScheduler.scala:833)\n",
       "  at org.apache.spark.scheduler.DAGScheduler$$anonfun$doCancelAllJobs$1.apply(DAGScheduler.scala:833)\n",
       "  at org.apache.spark.scheduler.DAGScheduler$$anonfun$doCancelAllJobs$1.apply(DAGScheduler.scala:833)\n",
       "  at scala.collection.mutable.HashSet.foreach(HashSet.scala:78)\n",
       "  at org.apache.spark.scheduler.DAGScheduler.doCancelAllJobs(DAGScheduler.scala:833)\n",
       "  at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2083)\n",
       "  at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2060)\n",
       "  at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2049)\n",
       "  at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)\n",
       "  at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:740)\n",
       "  at org.apache.spark.SparkContext.runJob(SparkContext.scala:2081)\n",
       "  at org.apache.spark.SparkContext.runJob(SparkContext.scala:2178)\n",
       "  at org.apache.spark.rdd.RDD$$anonfun$aggregate$1.apply(RDD.scala:1124)\n",
       "  at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)\n",
       "  at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)\n",
       "  at org.apache.spark.rdd.RDD.withScope(RDD.scala:363)\n",
       "  at org.apache.spark.rdd.RDD.aggregate(RDD.scala:1117)\n",
       "  at org.apache.spark.sql.execution.datasources.csv.CSVInferSchema$.infer(CSVInferSchema.scala:44)\n",
       "  at org.apache.spark.sql.execution.datasources.csv.TextInputCSVDataSource$.inferFromDataset(CSVDataSource.scala:257)\n",
       "  at org.apache.spark.sql.execution.datasources.csv.TextInputCSVDataSource$.infer(CSVDataSource.scala:233)\n",
       "  at org.apache.spark.sql.execution.datasources.csv.CSVDataSource.inferSchema(CSVDataSource.scala:68)\n",
       "  at org.apache.spark.sql.execution.datasources.csv.CSVFileFormat.inferSchema(CSVFileFormat.scala:63)\n",
       "  at org.apache.spark.sql.execution.datasources.DataSource$$anonfun$6.apply(DataSource.scala:179)\n",
       "  at org.apache.spark.sql.execution.datasources.DataSource$$anonfun$6.apply(DataSource.scala:179)\n",
       "  at scala.Option.orElse(Option.scala:289)\n",
       "  at org.apache.spark.sql.execution.datasources.DataSource.getOrInferFileFormatSchema(DataSource.scala:178)\n",
       "  at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:372)\n",
       "  at org.apache.spark.sql.DataFrameReader.loadV1Source(DataFrameReader.scala:223)\n",
       "  at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:211)\n",
       "  at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:178)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val magPapers = spark.read.format(\"csv\")\n",
    "      //.option(\"header\", \"true\")\n",
    "      .option(\"delimiter\",\"\\t\")\n",
    "      .option(\"inferSchema\",\"true\")\n",
    "      .load(\"wasbs://mag-2020-09-01@iunimag.blob.core.windows.net/mag/Papers.txt\")\n",
    "magPapers.show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+--------+-----+--------+------+------+-----+---------+-------+--------+------------+-------+-------+------+----------------+--------+--------------+--------------+-----+------+-----+-----+-----+-----+-----+-----+---+-----+-----+-----+-----+----+----+----+----+-----+-----+-----+-----+-----+-----+----+\n",
      "|summary|cord_uid|  sha|source_x| title|   doi|pmcid|pubmed_id|license|abstract|publish_time|authors|journal|mag_id|who_covidence_id|arxiv_id|pdf_json_files|pmc_json_files|  url| s2_id|  _c0|  _c1|  _c2|  _c3|  _c4|  _c5|_c6|  _c7|  _c8|  _c9| _c10|_c11|_c12|_c13|_c14| _c15| _c16| _c17| _c18| _c19| _c20|_c21|\n",
      "+-------+--------+-----+--------+------+------+-----+---------+-------+--------+------------+-------+-------+------+----------------+--------+--------------+--------------+-----+------+-----+-----+-----+-----+-----+-----+---+-----+-----+-----+-----+----+----+----+----+-----+-----+-----+-----+-----+-----+----+\n",
      "|  count|  128648|55847|  128648|128620|100773|60927|    99169| 128609|  101709|      128604| 123809| 122558|  2630|           19018|    3408|         57413|         45346|50465|101777|14945|14945|14945|12295|14945|14945|546|14945|14945|14697|11736| 332|  48|7724|6108|11011|10165|14945|14945|14945|12446| 345|\n",
      "+-------+--------+-----+--------+------+------+-----+---------+-------+--------+------------+-------+-------+------+----------------+--------+--------------+--------------+-----+------+-----+-----+-----+-----+-----+-----+---+-----+-----+-----+-----+----+----+----+----+-----+-----+-----+-----+-----+-----+----+\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "doiMatch = [cord_uid: string, sha: string ... 40 more fields]\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "source": "user"
    },
    {
     "data": {
      "text/plain": [
       "[cord_uid: string, sha: string ... 40 more fields]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val doiMatch = sdf.join(magPapers, sdf(\"doi\")===magPapers(\"_c2\"), \"left\").cache()\n",
    "doiMatch.describe().filter($\"summary\" === \"count\").show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+----------+--------+-----+--------+------+------+------+---------+-------+--------+------------+-------+-------+------+----------------+--------+--------------+--------------+------+------+----------+--------------+----------+--------------+-----+-----+-----------+------+\n",
      "|summary|Unnamed: 0|cord_uid|  sha|source_x| title|   doi| pmcid|pubmed_id|license|abstract|publish_time|authors|journal|mag_id|who_covidence_id|arxiv_id|pdf_json_files|pmc_json_files|   url| s2_id|pmcAuthors|pmcAffiliation|pdfAuthors|pdfAffiliation| pmid| year|annotations| MAGid|\n",
      "+-------+----------+--------+-----+--------+------+------+------+---------+-------+--------+------------+-------+-------+------+----------------+--------+--------------+--------------+------+------+----------+--------------+----------+--------------+-----+-----+-----------+------+\n",
      "|  count|    244035|  244035|95650|  244035|243977|160186|102577|   140745| 244035|  172927|      243978| 236700| 229504|     0|           69316|    3202|         95650|         74751|175015|212654|     74751|         74751|     90688|         90688|67696|67667|      67696|211355|\n",
      "+-------+----------+--------+-----+--------+------+------+------+---------+-------+--------+------------+-------+-------+------+----------------+--------+--------------+--------------+------+------+----------+--------------+----------+--------------+-----+-----+-----------+------+\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "idMap = [_c0: string, _c1: string]\n",
       "idMatch = [Unnamed: 0: string, cord_uid: string ... 26 more fields]\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "source": "user"
    },
    {
     "data": {
      "text/plain": [
       "lastException: Throwable = null\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "source": "user"
    },
    {
     "data": {
      "text/plain": [
       "[Unnamed: 0: string, cord_uid: string ... 26 more fields]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val idMap = spark.read.format(\"csv\").option(\"header\", false)\n",
    "            .load(\"wasbs://mag-2020-09-01@iunimag.blob.core.windows.net/2020-09-01-CORD-UID-MappedTo-2020-08-24-MAG-ID.csv\")\n",
    "val idMatch = sdf.join(idMap, sdf(\"cord_uid\")===idMap(\"_c0\"), \"left\").withColumnRenamed(\"_c1\",\"MAGid\").drop(\"_c0\")\n",
    "idMatch.describe().filter($\"summary\" === \"count\").show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+--------+------+\n",
      "|summary|cord_uid|MAGids|\n",
      "+-------+--------+------+\n",
      "|  count|  230834|230834|\n",
      "+-------+--------+------+\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "idMatch2 = [cord_uid: string, MAGids: string]\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "source": "user"
    },
    {
     "data": {
      "text/plain": [
       "[cord_uid: string, MAGids: string]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val idMatch2 = idMatch.groupBy(\"cord_uid\").agg(concat_ws(\";\", collect_list(\"MAGid\")) as \"MAGids\")\n",
    "idMatch2.describe().filter($\"summary\" === \"count\").show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Name: Compile Error\n",
       "Message: <console>:44: error: not found: value magPapers\n",
       "       val magMatch = idMatch.join(magPapers, idMatch(\"MAGid\")===magPapers(\"_c0\"), \"left\").cache()\n",
       "                                   ^\n",
       "<console>:44: error: not found: value magPapers\n",
       "       val magMatch = idMatch.join(magPapers, idMatch(\"MAGid\")===magPapers(\"_c0\"), \"left\").cache()\n",
       "                                                                 ^\n",
       "\n",
       "StackTrace: "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val magMatch = idMatch.join(magPapers, idMatch(\"MAGid\")===magPapers(\"_c0\"), \"left\").cache()\n",
    "magMatch.describe().filter($\"summary\" === \"count\").show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+----------+----------+---+--------------------+--------------------+\n",
      "|_c0|       _c1|       _c2|_c3|                 _c4|                 _c5|\n",
      "+---+----------+----------+---+--------------------+--------------------+\n",
      "|  9|2632942543|      null|  1|Victoriano Perruc...|                null|\n",
      "| 15| 199142497|      null|  1|     Robert Münscher|Heidelberg, Deuts...|\n",
      "| 15| 680395887|      null|  2|       Julia Hormuth|Reutlingen, Deuts...|\n",
      "| 23|1243978490|  79576946|  1|     Eric T Stoopler|Department of Ora...|\n",
      "| 23|2582258949|      null|  3| Arthur S Kuperstein|                null|\n",
      "| 23|2582736345|      null|  2|        Ying Wai Sia|                null|\n",
      "| 58|2641488431|      null|  1|             I. Uray|                null|\n",
      "| 58|2693978806|      null|  2|           P. Kovacs|                null|\n",
      "| 79|2662843304|      null|  1|           K. Suresh|                null|\n",
      "| 79|2683440697|      null|  2|     T. S. J. Norman|                null|\n",
      "| 87|2676159639|      null|  1| Gabriel Torres Puga|                null|\n",
      "|108|2126642415| 165102784|  1|      Matthew Lovett| Duquesne University|\n",
      "|125|1995014452| 169199633|  2|     David Hannaford|Aston University ...|\n",
      "|125|2002579779| 169199633|  1|           Jon Bader|Aston University ...|\n",
      "|125|2250382311| 169199633|  4|        John Edwards|Aston University ...|\n",
      "|125|2283694448| 169199633|  3|    Alastair Cochran|Aston University ...|\n",
      "|138|2729948773|1316910977|  2|       Tropea Alfred|E.I. DU PONT DE N...|\n",
      "|138|2825090221|      null|  1|      Blackwell John|CROMPTON & KNOWLE...|\n",
      "|143|2137699977|      null|  1|     Lina Birgersson|                null|\n",
      "|147|2616864840|      null|  1|       Callejas Pozo|                null|\n",
      "+---+----------+----------+---+--------------------+--------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "magPPA = [_c0: bigint, _c1: bigint ... 4 more fields]\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "source": "user"
    },
    {
     "data": {
      "text/plain": [
       "[_c0: bigint, _c1: bigint ... 4 more fields]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val magPPA = spark.read.format(\"csv\")\n",
    "      //.option(\"header\", \"true\")\n",
    "      .option(\"delimiter\",\"\\t\")\n",
    "      .option(\"inferSchema\",\"true\")\n",
    "      .load(\"wasbs://mag-2020-09-01@iunimag.blob.core.windows.net/mag/PaperAuthorAffiliations.txt\")\n",
    "\n",
    "magPPA.show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+----------+--------+------+--------+------+------+------+---------+-------+--------+------------+-------+-------+------+----------------+--------+--------------+--------------+------+------+----------+--------------+----------+--------------+------+------+-----------+------+------+--------+-------------+-----------+------+-----------+\n",
      "|summary|Unnamed: 0|cord_uid|   sha|source_x| title|   doi| pmcid|pubmed_id|license|abstract|publish_time|authors|journal|mag_id|who_covidence_id|arxiv_id|pdf_json_files|pmc_json_files|   url| s2_id|pmcAuthors|pmcAffiliation|pdfAuthors|pdfAffiliation|  pmid|  year|annotations| MAGid|MAGid2|authorID|affiliationID|authorOrder|author|affiliation|\n",
      "+-------+----------+--------+------+--------+------+------+------+---------+-------+--------+------------+-------+-------+------+----------------+--------+--------------+--------------+------+------+----------+--------------+----------+--------------+------+------+-----------+------+------+--------+-------------+-----------+------+-----------+\n",
      "|  count|    924303|  924303|443329|  924303|924231|708263|431099|   628274| 924303|  750099|      924150| 909268| 868530|     0|          186528|    6494|        443329|        341356|738701|821175|    341356|        341356|    409456|        409456|252964|252837|     252964|891623|848934|  848934|       848934|     848934|848934|     808050|\n",
      "+-------+----------+--------+------+--------+------+------+------+---------+-------+--------+------------+-------+-------+------+----------------+--------+--------------+--------------+------+------+----------+--------------+----------+--------------+------+------+-----------+------+------+--------+-------------+-----------+------+-----------+\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "magPPA2 = [MAGid2: bigint, authorID: bigint ... 4 more fields]\n",
       "magMatchPPA = [Unnamed: 0: string, cord_uid: string ... 32 more fields]\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "source": "user"
    },
    {
     "data": {
      "text/plain": [
       "[Unnamed: 0: string, cord_uid: string ... 32 more fields]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val magPPA2 = magPPA.toDF(\"MAGid2\",\"authorID\",\"affiliationID\",\"authorOrder\",\"author\",\"affiliation\").filter(\"affiliationID is not null\")\n",
    "val magMatchPPA = idMatch.join(magPPA2, idMatch(\"MAGid\")===magPPA2(\"MAGid2\"), \"left\").cache()\n",
    "magMatchPPA.describe().filter($\"summary\" === \"count\").show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+--------------------+--------------------+--------------------+-------------+--------------------+------------------+------------------+-----------------+-----------+------------------+------------------+\n",
      "|summary|      AffiliationId2|      normalizedName|         displayName|       GRIDid|                wiki|        paperTotal|  paperFamilyTotal|    citationTotal|Iso3166Code|          Latitude|         Longitude|\n",
      "+-------+--------------------+--------------------+--------------------+-------------+--------------------+------------------+------------------+-----------------+-----------+------------------+------------------+\n",
      "|  count|               25785|               25785|               25785|        19026|               25496|             25785|             25785|            25785|      24689|             24790|             24790|\n",
      "|   mean| 1.076605067649137E9|                null|                null|         null|                null|3316.6961023851077|3263.5179367849523|71568.24886561955|       null|34.607002906087146|-7.433453639265983|\n",
      "| stddev|1.1731447217708974E9|                null|                null|         null|                null|15891.314883906729|15574.676129772803|490832.4225437315|       null| 19.37216884102037| 82.46863265476821|\n",
      "|    min|                4605|                  3m|\"Universitas Putr...|  grid.1001.0|http://blog.seman...|                 1|                 1|                0|         AE|       -53.1363945|       -171.750336|\n",
      "|    max|          3081383786|ştefan cel mare u...|​Mercy College of...|grid.99921.3a|https://www.wikid...|            577585|            566667|         27328739|         ZW|          78.23587|           178.448|\n",
      "+-------+--------------------+--------------------+--------------------+-------------+--------------------+------------------+------------------+-----------------+-----------+------------------+------------------+\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "magAffiliation = [_c0: bigint, _c1: int ... 12 more fields]\n",
       "magAffiliation2 = [AffiliationId2: bigint, normalizedName: string ... 9 more fields]\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "source": "user"
    },
    {
     "data": {
      "text/plain": [
       "[AffiliationId2: bigint, normalizedName: string ... 9 more fields]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val magAffiliation = spark.read.format(\"csv\")\n",
    "      //.option(\"header\", \"true\")\n",
    "      .option(\"delimiter\",\"\\t\")\n",
    "      .option(\"inferSchema\",\"true\")\n",
    "      .load(\"wasbs://mag-2020-09-01@iunimag.blob.core.windows.net/mag/Affiliations.txt\")\n",
    "\n",
    "val magAffiliation2 = magAffiliation.select(\"_c0\",\"_c2\",\"_c3\",\"_c4\",\"_c6\",\"_c7\",\"_c8\",\"_c9\",\"_c10\",\"_c11\",\"_c12\").toDF(\"AffiliationId2\",\"normalizedName\",\"displayName\",\"GRIDid\",\"wiki\",\"paperTotal\",\"paperFamilyTotal\",\"citationTotal\", \"Iso3166Code\", \"Latitude\", \"Longitude\")\n",
    "magAffiliation2.describe().show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+----------+--------+------+--------+------+------+------+---------+-------+--------+------------+-------+-------+------+----------------+--------+--------------+--------------+------+------+----------+--------------+----------+--------------+------+------+-----------+------+------+--------+-------------+-----------+------+-----------+--------------+--------------+-----------+------+------+----------+-------------+--------+---------+\n",
      "|summary|Unnamed: 0|cord_uid|   sha|source_x| title|   doi| pmcid|pubmed_id|license|abstract|publish_time|authors|journal|mag_id|who_covidence_id|arxiv_id|pdf_json_files|pmc_json_files|   url| s2_id|pmcAuthors|pmcAffiliation|pdfAuthors|pdfAffiliation|  pmid|  year|annotations| MAGid|MAGid2|authorID|affiliationID|authorOrder|author|affiliation|affiliationID2|normalizedName|displayName|GRIDid|  wiki|paperTotal|citationTotal|Latitude|Longitude|\n",
      "+-------+----------+--------+------+--------+------+------+------+---------+-------+--------+------------+-------+-------+------+----------------+--------+--------------+--------------+------+------+----------+--------------+----------+--------------+------+------+-----------+------+------+--------+-------------+-----------+------+-----------+--------------+--------------+-----------+------+------+----------+-------------+--------+---------+\n",
      "|  count|    924303|  924303|443329|  924303|924231|708263|431099|   628274| 924303|  750099|      924150| 909268| 868530|     0|          186528|    6494|        443329|        341356|738701|821175|    341356|        341356|    409456|        409456|252964|252837|     252964|891623|848934|  848934|       848934|     848934|848934|     808050|        848934|        848934|     848934|806111|846597|    848934|       848934|  843321|   846854|\n",
      "+-------+----------+--------+------+--------+------+------+------+---------+-------+--------+------------+-------+-------+------+----------------+--------+--------------+--------------+------+------+----------+--------------+----------+--------------+------+------+-----------+------+------+--------+-------------+-----------+------+-----------+--------------+--------------+-----------+------+------+----------+-------------+--------+---------+\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "magAffiliation2 = [affiliationID2: bigint, normalizedName: string ... 7 more fields]\n",
       "magMatchPPA2 = [Unnamed: 0: string, cord_uid: string ... 41 more fields]\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "source": "user"
    },
    {
     "data": {
      "text/plain": [
       "[Unnamed: 0: string, cord_uid: string ... 41 more fields]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val magAffiliation2 = magAffiliation.select(\"_c0\",\"_c2\",\"_c3\",\"_c4\",\"_c6\",\"_c7\",\"_c9\",\"_c10\",\"_c11\").toDF(\"affiliationID2\",\"normalizedName\",\"displayName\",\"GRIDid\",\"wiki\",\"paperTotal\", \"citationTotal\", \"Latitude\", \"Longitude\")\n",
    "val magMatchPPA2 = magMatchPPA.join(magAffiliation2, magMatchPPA(\"affiliationID\")===magAffiliation2(\"affiliationID2\"), \"left\").cache()\n",
    "magMatchPPA2.describe().filter($\"summary\" === \"count\").show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+-------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------+--------------------+-------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "|cord_uid|MAGids                                                                                                       |authorids                                                                                                    |authorOrders        |affiliationids                                                                                               |affiliationNormalized                                                                                                                                                                                                                                                                                         |affiliationNames                                                                                                                                                                                                                                                                                                    |GRIDids                                                                                                                                    |\n",
      "+--------+-------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------+--------------------+-------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "|00a19z5i|2065863039;2065863039;2065863039;2065863039;2065863039;2065863039                                            |19913035;1998044908;2085272691;2103951155;2313423376;2491059991                                              |2;1;3;4;5;6         |2800006345;1299303238;1850255;1850255;1299303238;1299303238                                                  |leiden university medical center;national institutes of health;university of basel;university of basel;national institutes of health;national institutes of health                                                                                                                                            |Leiden University Medical Center;National Institutes of Health;University of Basel;University of Basel;National Institutes of Health;National Institutes of Health                                                                                                                                                  |grid.10419.3d;grid.94365.3d;grid.6612.3;grid.6612.3;grid.94365.3d;grid.94365.3d                                                            |\n",
      "|02rcmt0g|2012911450;2012911450;2012911450;2012911450;2012911450                                                       |2127844664;2152867687;2154212754;2429219040;2776001672                                                       |1;5;3;2;4           |592451;592451;592451;592451;31419693                                                                         |indiana university;indiana university;indiana university;indiana university;kyungpook national university                                                                                                                                                                                                     |Indiana University;Indiana University;Indiana University;Indiana University;Kyungpook National University                                                                                                                                                                                                           |grid.257410.5;grid.257410.5;grid.257410.5;grid.257410.5;grid.258803.4                                                                      |\n",
      "|034w5afv|2998409415;2998409415;2998409415;2998409415;2998409415;2998409415                                            |125278209;2012668690;2019881018;2315351325;2899416205;2997211881                                             |4;3;7;5;2;1         |884181516;884181516;884181516;884181516;884181516;884181516                                                  |universidad autonoma del estado de morelos;universidad autonoma del estado de morelos;universidad autonoma del estado de morelos;universidad autonoma del estado de morelos;universidad autonoma del estado de morelos;universidad autonoma del estado de morelos                                             |Universidad Autónoma del Estado de Morelos;Universidad Autónoma del Estado de Morelos;Universidad Autónoma del Estado de Morelos;Universidad Autónoma del Estado de Morelos;Universidad Autónoma del Estado de Morelos;Universidad Autónoma del Estado de Morelos                                                   |grid.412873.b;grid.412873.b;grid.412873.b;grid.412873.b;grid.412873.b;grid.412873.b                                                        |\n",
      "|03er0xjy|3047961869;3047961869;3047961869;3047961869;3047961869;3047961869;3047961869                                 |2016451659;2121565218;2627349505;2792035902;2965788016;2976439631;3048285090                                 |7;6;5;2;3;4;1       |139660479;139660479;139660479;139660479;139660479;139660479;139660479                                        |central south university;central south university;central south university;central south university;central south university;central south university;central south university                                                                                                                                |Central South University;Central South University;Central South University;Central South University;Central South University;Central South University;Central South University                                                                                                                                      |grid.216417.7;grid.216417.7;grid.216417.7;grid.216417.7;grid.216417.7;grid.216417.7;grid.216417.7                                          |\n",
      "|04uf62g7|2594827305                                                                                                   |2628219796                                                                                                   |3                   |2800491743                                                                                                   |sacred heart hospital                                                                                                                                                                                                                                                                                         |Sacred Heart Hospital                                                                                                                                                                                                                                                                                               |grid.460942.d                                                                                                                              |\n",
      "|07peindz|                                                                                                             |                                                                                                             |                    |                                                                                                             |                                                                                                                                                                                                                                                                                                              |                                                                                                                                                                                                                                                                                                                    |                                                                                                                                           |\n",
      "|08hc3bwd|3042066663;3042066663;3042066663;3042066663;3042066663                                                       |1978686820;2040088585;2178443833;2651447467;3019094123                                                       |12;13;8;9;10        |189158943;189158943;189158943;189158943;189158943                                                            |university of milan;university of milan;university of milan;university of milan;university of milan                                                                                                                                                                                                           |University of Milan;University of Milan;University of Milan;University of Milan;University of Milan                                                                                                                                                                                                                 |grid.4708.b;grid.4708.b;grid.4708.b;grid.4708.b;grid.4708.b                                                                                |\n",
      "|0bogmafi|                                                                                                             |                                                                                                             |                    |                                                                                                             |                                                                                                                                                                                                                                                                                                              |                                                                                                                                                                                                                                                                                                                    |                                                                                                                                           |\n",
      "|0bpse3a0|2054669441;2054669441;2054669441;2054669441;2054669441                                                       |2233198491;2275196589;2304836341;2809343818;2809343818                                                       |2;4;3;1;1           |124601658;124601658;124601658;124601658;1313323035                                                           |university of parma;university of parma;university of parma;university of parma;russian academy of sciences                                                                                                                                                                                                   |University of Parma;University of Parma;University of Parma;University of Parma;Russian Academy of Sciences                                                                                                                                                                                                         |grid.10383.39;grid.10383.39;grid.10383.39;grid.10383.39;grid.4886.2                                                                        |\n",
      "|0c8hob5l|2917228511;2917228511;2917228511;2917228511;2917228511;2917228511;2917228511;2917228511;2917228511;2917228511|2100982618;2103174634;2549886304;2619766754;2765815937;2766749557;2801314204;2912623302;2913416547;2913631469|10;1;9;4;3;2;7;8;5;6|1327930874;1327930874;1327930874;1327930874;1327930874;1327930874;1327930874;1327930874;1327930874;1327930874|apollo hospitals;apollo hospitals;apollo hospitals;apollo hospitals;apollo hospitals;apollo hospitals;apollo hospitals;apollo hospitals;apollo hospitals;apollo hospitals                                                                                                                                     |Apollo Hospitals;Apollo Hospitals;Apollo Hospitals;Apollo Hospitals;Apollo Hospitals;Apollo Hospitals;Apollo Hospitals;Apollo Hospitals;Apollo Hospitals;Apollo Hospitals                                                                                                                                           |grid.413839.4;grid.413839.4;grid.413839.4;grid.413839.4;grid.413839.4;grid.413839.4;grid.413839.4;grid.413839.4;grid.413839.4;grid.413839.4|\n",
      "|0dac26xk|                                                                                                             |                                                                                                             |                    |                                                                                                             |                                                                                                                                                                                                                                                                                                              |                                                                                                                                                                                                                                                                                                                    |                                                                                                                                           |\n",
      "|0egc0hkx|3035520492                                                                                                   |3006491464                                                                                                   |1                   |18729865                                                                                                     |british dental association                                                                                                                                                                                                                                                                                    |British Dental Association                                                                                                                                                                                                                                                                                          |                                                                                                                                           |\n",
      "|0f5d2lxy|3042807636;3042807636;3042807636                                                                             |2755134825;2902205447;3042384681                                                                             |3;1;2               |181190671;181190671;181190671                                                                                |new york medical college;new york medical college;new york medical college                                                                                                                                                                                                                                    |New York Medical College;New York Medical College;New York Medical College                                                                                                                                                                                                                                          |grid.260917.b;grid.260917.b;grid.260917.b                                                                                                  |\n",
      "|0hf5nkbc|3023025232;3023025232;3023025232;3023025232;3023025232;3023025232;3023025232                                 |2132439200;2635336135;2670433135;2801183877;2964955867;3020821853;3022511426                                 |9;5;4;6;1;3;7       |20231570;20231570;20231570;20231570;20231570;2802939634;2802939634                                           |peking university;peking university;peking university;peking university;peking university;chinese pla general hospital;chinese pla general hospital                                                                                                                                                           |Peking University;Peking University;Peking University;Peking University;Peking University;Chinese PLA General Hospital;Chinese PLA General Hospital                                                                                                                                                                 |grid.11135.37;grid.11135.37;grid.11135.37;grid.11135.37;grid.11135.37;grid.414252.4;grid.414252.4                                          |\n",
      "|0if5z3xp|2064570507;2064570507                                                                                        |2045917102;2110523902                                                                                        |2;1                 |141945490;141945490                                                                                          |university of british columbia;university of british columbia                                                                                                                                                                                                                                                 |University of British Columbia;University of British Columbia                                                                                                                                                                                                                                                       |grid.17091.3e;grid.17091.3e                                                                                                                |\n",
      "|0ip6g4pj|3049546925;3049546925                                                                                        |2150369999;3041757481                                                                                        |1;2                 |280994;280994                                                                                                |king george s medical university;king george s medical university                                                                                                                                                                                                                                             |King George's Medical University;King George's Medical University                                                                                                                                                                                                                                                   |grid.411275.4;grid.411275.4                                                                                                                |\n",
      "|0jkl2f66|2568259424;2568259424;2568259424;2568259424                                                                  |2171645175;2178826165;2587886194;2588078072                                                                  |4;3;1;2             |165951966;165951966;165951966;1289568071                                                                     |university of texas health science center at san antonio;university of texas health science center at san antonio;university of texas health science center at san antonio;san antonio military medical center                                                                                                |University of Texas Health Science Center at San Antonio;University of Texas Health Science Center at San Antonio;University of Texas Health Science Center at San Antonio;San Antonio Military Medical Center                                                                                                      |grid.267309.9;grid.267309.9;grid.267309.9;grid.416653.3                                                                                    |\n",
      "|0k0n62rm|                                                                                                             |                                                                                                             |                    |                                                                                                             |                                                                                                                                                                                                                                                                                                              |                                                                                                                                                                                                                                                                                                                    |                                                                                                                                           |\n",
      "|0lv2d091|2531686969;2531686969;2531686969;2531686969;2531686969;2531686969;2531686969;2531686969;2531686969;2531686969|2100114044;2100166561;2100166561;2165917120;2184489236;2305222930;2491336121;2551485194;2552766008;2572735316|9;5;5;6;1;2;3;4;7;8 |68171755;91656880;151221077;16733864;68171755;16733864;68171755;68171755;68171755;68171755                   |national defense medical center;china medical university;chung yuan christian university;national taiwan university;national defense medical center;national taiwan university;national defense medical center;national defense medical center;national defense medical center;national defense medical center|National Defense Medical Center;China Medical University (PRC);Chung Yuan Christian University;National Taiwan University;National Defense Medical Center;National Taiwan University;National Defense Medical Center;National Defense Medical Center;National Defense Medical Center;National Defense Medical Center|grid.260565.2;grid.412449.e;grid.411649.f;grid.19188.39;grid.260565.2;grid.19188.39;grid.260565.2;grid.260565.2;grid.260565.2;grid.260565.2|\n",
      "|0mv3578z|3044937103                                                                                                   |3016201764                                                                                                   |1                   |47720641                                                                                                     |huazhong university of science and technology                                                                                                                                                                                                                                                                 |Huazhong University of Science and Technology                                                                                                                                                                                                                                                                       |grid.33199.31                                                                                                                              |\n",
      "+--------+-------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------+--------------------+-------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "PaperMAG = [cord_uid: string, MAGids: string ... 6 more fields]\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "source": "user"
    },
    {
     "data": {
      "text/plain": [
       "[cord_uid: string, MAGids: string ... 6 more fields]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val PaperMAG = magMatchPPA2.groupBy(\"cord_uid\").agg(concat_ws(\";\", collect_list(\"MAGid2\")) as \"MAGids\",\n",
    "                                                    concat_ws(\";\", collect_list(\"authorID\")) as \"authorids\",\n",
    "                                                    concat_ws(\";\", collect_list(\"authorOrder\")) as \"authorOrders\",\n",
    "                                                    concat_ws(\";\", collect_list(\"affiliationID2\")) as \"affiliationids\",\n",
    "                                                    concat_ws(\";\", collect_list(\"normalizedName\")) as \"affiliationNormalized\",\n",
    "                                                    concat_ws(\";\", collect_list(\"displayName\")) as \"affiliationNames\",\n",
    "                                                    concat_ws(\";\", collect_list(\"GRIDid\")) as \"GRIDids\")\n",
    "PaperMAG.show(false)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+--------+----------+-----+--------+------+------+------+---------+-------+--------+------------+-------+-------+------+----------------+--------+--------------+--------------+------+------+----------+--------------+----------+--------------+-----+-----+-----------+------+---------+------------+--------------+---------------------+----------------+-------+\n",
      "|summary|cord_uid|Unnamed: 0|  sha|source_x| title|   doi| pmcid|pubmed_id|license|abstract|publish_time|authors|journal|mag_id|who_covidence_id|arxiv_id|pdf_json_files|pmc_json_files|   url| s2_id|pmcAuthors|pmcAffiliation|pdfAuthors|pdfAffiliation| pmid| year|annotations|MAGids|authorids|authorOrders|affiliationids|affiliationNormalized|affiliationNames|GRIDids|\n",
      "+-------+--------+----------+-----+--------+------+------+------+---------+-------+--------+------------+-------+-------+------+----------------+--------+--------------+--------------+------+------+----------+--------------+----------+--------------+-----+-----+-----------+------+---------+------------+--------------+---------------------+----------------+-------+\n",
      "|  count|  242164|    242164|94862|  242164|242108|158784|101673|   139396| 242164|  171771|      242107| 234852| 227648|     0|           68828|    3197|         94862|         74137|173596|210817|     74137|         74137|     89969|         89969|65978|65949|      65978|242164|   242164|      242164|        242164|               242164|          242164| 242164|\n",
      "+-------+--------+----------+-----+--------+------+------+------+---------+-------+--------+------------+-------+-------+------+----------------+--------+--------------+--------------+------+------+----------+--------------+----------+--------------+-----+-----+-----------+------+---------+------------+--------------+---------------------+----------------+-------+\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "PaperMAG2 = [cord_uid: string, Unnamed: 0: string ... 32 more fields]\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "source": "user"
    },
    {
     "data": {
      "text/plain": [
       "[cord_uid: string, Unnamed: 0: string ... 32 more fields]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val PaperMAG2 = sdf.join(PaperMAG, Seq(\"cord_uid\"), \"left\").drop(\"_c0\").cache()\n",
    "PaperMAG2.describe().filter($\"summary\" === \"count\").show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "PaperMAG2.coalesce(1).write.mode(SaveMode.Overwrite)\n",
    "    .format(\"com.databricks.spark.csv\")\n",
    "    .option(\"header\", \"true\")\n",
    "    .option(\"quote\", \"\\\"\")\n",
    "    .save(\"wasbs://mag-2020-09-01@iunimag.blob.core.windows.net/2020-08-03/CORD-19-MAGinstsFull.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------+---------------------------------------------------+--------------------------------------------------+--------------------------------------------------------------------------------+----------+-------------+--------+-----------+-------------+\n",
      "|affiliationID|displayName                                        |normalizedName                                    |wiki                                                                            |paperTotal|citationTotal|Latitude|Longitude  |GRIDid       |\n",
      "+-------------+---------------------------------------------------+--------------------------------------------------+--------------------------------------------------------------------------------+----------+-------------+--------+-----------+-------------+\n",
      "|746986       |Halmstad University                                |halmstad university                               |http://en.wikipedia.org/wiki/Halmstad_University,_Sweden                        |2548      |39177        |SE      |56.66417   |grid.73638.39|\n",
      "|6141010      |European Institute of Oncology                     |european institute of oncology                    |http://en.wikipedia.org/wiki/European_Institute_of_Oncology                     |7874      |298771       |IT      |45.4140358 |grid.15667.33|\n",
      "|8408910      |Cardiff Metropolitan University                    |cardiff metropolitan university                   |http://en.wikipedia.org/wiki/Cardiff_Metropolitan_University                    |2664      |42481        |GB      |51.4843    |grid.47170.35|\n",
      "|10108838     |National University of Salta                       |national university of salta                      |http://en.wikipedia.org/wiki/National_University_of_Salta                       |165       |1299         |AR      |-24.7864914|grid.10821.3a|\n",
      "|23171815     |Zhengzhou University of Light Industry             |zhengzhou university of light industry            |http://en.wikipedia.org/wiki/Zhengzhou_University_of_Light_Industry             |5588      |34687        |CN      |34.79742   |grid.413080.e|\n",
      "|45438204     |University of Texas at San Antonio                 |university of texas at san antonio                |http://en.wikipedia.org/wiki/University_of_Texas_at_San_Antonio                 |22833     |541199       |US      |29.59      |grid.215352.2|\n",
      "|47396830     |Celgene                                            |celgene                                           |http://en.wikipedia.org/wiki/Celgene                                            |3372      |80451        |US      |40.70987   |grid.488233.6|\n",
      "|49185572     |Yunnan University of Finance and Economics         |yunnan university of finance and economics        |http://en.wikipedia.org/wiki/Yunnan_University_of_Finance_and_Economics         |2089      |7568         |CN      |24.9266472 |grid.464506.5|\n",
      "|51195161     |EMLYON Business School                             |emlyon business school                            |http://en.wikipedia.org/wiki/EMLYON_Business_School                             |1478      |22919        |FR      |45.78611   |grid.462218.b|\n",
      "|53298266     |Belarusian State Medical University                |belarusian state medical university               |http://en.wikipedia.org/wiki/Belarusian_State_Medical_University                |559       |1821         |BY      |53.86139   |grid.21354.31|\n",
      "|55486353     |Reed College                                       |reed college                                      |http://en.wikipedia.org/wiki/Reed_College                                       |2156      |52972        |US      |45.48      |grid.182981.b|\n",
      "|57496824     |National Institute of Technology, Arunachal Pradesh|national institute of technology arunachal pradesh|http://en.wikipedia.org/wiki/National_Institute_of_Technology,_Arunachal_Pradesh|268       |556          |IN      |27.1498    |grid.464634.7|\n",
      "|60054993     |Graphic Era University                             |graphic era university                            |http://en.wikipedia.org/wiki/Graphic_Era_University                             |1049      |5557         |IN      |30.26747   |grid.448909.8|\n",
      "|61057504     |Fujian Agriculture and Forestry University         |fujian agriculture and forestry university        |http://en.wikipedia.org/wiki/Fujian_Agriculture_and_Forestry_University         |12343     |70276        |CN      |26.08778   |grid.256111.0|\n",
      "|67357951     |KIIT University                                    |kiit university                                   |http://en.wikipedia.org/wiki/KIIT_University                                    |4001      |22316        |IN      |20.35278   |grid.412122.6|\n",
      "|72054575     |Cochrane Collaboration                             |cochrane collaboration                            |http://en.wikipedia.org/wiki/Cochrane_(organisation)                            |4139      |190523       |GB      |51.7791138 |grid.420305.0|\n",
      "|75124487     |Gdańsk Medical University                          |gdansk medical university                         |http://en.wikipedia.org/wiki/Gdańsk_Medical_University                          |10046     |174658       |PL      |54.36611   |grid.11451.30|\n",
      "|80849659     |National Institute for Space Research              |national institute for space research             |http://en.wikipedia.org/wiki/National_Institute_for_Space_Research              |7127      |109889       |BR      |-23.2070217|grid.419222.e|\n",
      "|91993116     |SIRRIS                                             |sirris                                            |http://en.wikipedia.org/wiki/SIRRIS                                             |137       |1020         |BE      |50.85107   |grid.423712.1|\n",
      "|111575329    |Bharathiar University                              |bharathiar university                             |http://en.wikipedia.org/wiki/Bharathiar_University                              |7831      |94958        |IN      |11.0397692 |grid.411677.2|\n",
      "+-------------+---------------------------------------------------+--------------------------------------------------+--------------------------------------------------------------------------------+----------+-------------+--------+-----------+-------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "insititutionMAG = [affiliationID: bigint, displayName: string ... 7 more fields]\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "source": "user"
    },
    {
     "data": {
      "text/plain": [
       "[affiliationID: bigint, displayName: string ... 7 more fields]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val insititutionMAG = magMatchPPA2.groupBy(\"affiliationID\").agg(//concat_ws(\";\", collect_list(\"MAGid2\")) as \"MAGids\",\n",
    "                                                    //concat_ws(\";\", collect_list(\"cord_uid\")) as \"cord_uids\",\n",
    "                                                    first(\"displayName\") as \"displayName\",\n",
    "                                                    first(\"normalizedName\") as \"normalizedName\",\n",
    "                                                    first(\"wiki\") as \"wiki\",\n",
    "                                                    first(\"paperTotal\") as \"paperTotal\", \n",
    "                                                    first(\"citationTotal\") as \"citationTotal\",\n",
    "                                                    first(\"Latitude\") as \"Latitude\",\n",
    "                                                    first(\"Longitude\") as \"Longitude\",\n",
    "                                                    first(\"GRIDid\") as \"GRIDid\")\n",
    "insititutionMAG.show(false)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "insititutionMAG//.select(\"wosid\",\"headings\",\"subheadings\",\"WC\",\"SC\",\"disciplines\",\"subdisciplines\",\"fractions\")\n",
    "    .coalesce(1).write.mode(SaveMode.Overwrite)\n",
    "    .format(\"com.databricks.spark.csv\")\n",
    "    .option(\"header\", \"true\")\n",
    "    .option(\"quote\", \"\\\"\")\n",
    "    .save(\"wasbs://mag-2020-09-01@iunimag.blob.core.windows.net/2020-08-03/MAGInstTable.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------+--------+-----+--------+------+------+-----+---------+-------+--------+------------+-------+-------+------+----------------+--------+--------------+--------------+------+------+----------+----------+---------+--------------+--------------+------+\n",
      "|summary|   _c0|cord_uid|  sha|source_x| title|   doi|pmcid|pubmed_id|license|abstract|publish_time|authors|journal|mag_id|who_covidence_id|arxiv_id|pdf_json_files|pmc_json_files|   url| s2_id|pmcAuthors|pdfAuthors|SSAuthors|pmcAffiliation|pdfAffiliation|SSRefs|\n",
      "+-------+------+--------+-----+--------+------+------+-----+---------+-------+--------+------------+-------+-------+------+----------------+--------+--------------+--------------+------+------+----------+----------+---------+--------------+--------------+------+\n",
      "|  count|137411|  137411|63458|  137411|137370|113076|70527|   100629| 137359|  105932|      137366| 132106| 130231|  2649|           27941|    3719|         64965|         52370|124586|111762|     52315|     61451|    64429|         52685|         61040| 64081|\n",
      "+-------+------+--------+-----+--------+------+------+-----+---------+-------+--------+------------+-------+-------+------+----------------+--------+--------------+--------------+------+------+----------+----------+---------+--------------+--------------+------+\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "meta0b = [_c0: int, cord_uid: string ... 24 more fields]\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "source": "user"
    },
    {
     "data": {
      "text/plain": [
       "[_c0: int, cord_uid: string ... 24 more fields]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val meta0b = spark.read.format(\"csv\")\n",
    "      .option(\"header\", \"true\")\n",
    "      //.option(\"delimiter\",\"\\t\")\n",
    "      .option(\"inferSchema\",\"true\")\n",
    "      .load(\"/dataRCSC/metaAuthors.csv\")\n",
    "\n",
    "meta0b.describe().filter($\"summary\" === \"count\").show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+--------+---+--------+-----+----+-----+---------+-------+--------+------------+-------+-------+------+----------------+--------+--------------+--------------+---+-----+----------+--------------+----------+--------------+----+-----------+------+---------+------------+--------------+---------------------+----------------+-------+--------+------+----+----------+----------+-------------+--------------+-------------+-----------+------------+\n",
      "|summary|cord_uid|sha|source_x|title| doi|pmcid|pubmed_id|license|abstract|publish_time|authors|journal|mag_id|who_covidence_id|arxiv_id|pdf_json_files|pmc_json_files|url|s2_id|pmcAuthors|pmcAffiliation|pdfAuthors|pdfAffiliation|pmid|annotations|MAGids|authorids|authorOrders|affiliationids|affiliationNormalized|affiliationNames|GRIDids|xref_doi|art_no| UID|WoSauthors|WoSjournal|score_title_0|score_author_0|ID_researcher|Author_name|author_order|\n",
      "+-------+--------+---+--------+-----+----+-----+---------+-------+--------+------------+-------+-------+------+----------------+--------+--------------+--------------+---+-----+----------+--------------+----------+--------------+----+-----------+------+---------+------------+--------------+---------------------+----------------+-------+--------+------+----+----------+----------+-------------+--------------+-------------+-----------+------------+\n",
      "|  count|    5863|415|    5863| 5863|3172|  559|      246|   5863|    3431|        5859|   5863|   5659|     0|            5272|       3|           415|           299|597| 5556|       299|           299|       401|           401|5528|       1641|  3220|     3220|        3220|          3220|                 3220|            3220|   3112|    1889|   479|5863|      5863|      5863|         5863|          5863|         5863|       5863|        5863|\n",
      "+-------+--------+---+--------+-----+----+-----+---------+-------+--------+------------+-------+-------+------+----------------+--------+--------------+--------------+---+-----+----------+--------------+----------+--------------+----+-----------+------+---------+------------+--------------+---------------------+----------------+-------+--------+------+----+----------+----------+-------------+--------------+-------------+-----------+------------+\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "meta0 = [cord_uid: string, sha: string ... 40 more fields]\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "source": "user"
    },
    {
     "data": {
      "text/plain": [
       "[cord_uid: string, sha: string ... 40 more fields]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val meta0 = spark.read.format(\"csv\")\n",
    "    .option(\"header\", \"true\")\n",
    "    .option(\"quote\", \"\\\"\")\n",
    "    .option(\"escape\", \"\\\"\")\n",
    "    //.option(\"inferSchema\",\"true\")\n",
    "    .load(\"wasbs://mag-2020-09-01@iunimag.blob.core.windows.net/testData.csv\")\n",
    "meta0.describe().filter($\"summary\" === \"count\").show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+--------+-----+\n",
      "|summary|cord_uid|count|\n",
      "+-------+--------+-----+\n",
      "|  count|    4989| 4989|\n",
      "+-------+--------+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "meta0.groupBy(\"cord_uid\").count().describe().filter($\"summary\" === \"count\").show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- authors: array (nullable = true)\n",
      " |    |-- element: struct (containsNull = true)\n",
      " |    |    |-- ids: array (nullable = true)\n",
      " |    |    |    |-- element: string (containsNull = true)\n",
      " |    |    |-- name: string (nullable = true)\n",
      " |-- doi: string (nullable = true)\n",
      " |-- doiUrl: string (nullable = true)\n",
      " |-- entities: array (nullable = true)\n",
      " |    |-- element: string (containsNull = true)\n",
      " |-- fieldsOfStudy: array (nullable = true)\n",
      " |    |-- element: string (containsNull = true)\n",
      " |-- id: string (nullable = true)\n",
      " |-- inCitations: array (nullable = true)\n",
      " |    |-- element: string (containsNull = true)\n",
      " |-- journalName: string (nullable = true)\n",
      " |-- journalPages: string (nullable = true)\n",
      " |-- journalVolume: string (nullable = true)\n",
      " |-- magId: string (nullable = true)\n",
      " |-- outCitations: array (nullable = true)\n",
      " |    |-- element: string (containsNull = true)\n",
      " |-- paperAbstract: string (nullable = true)\n",
      " |-- pdfUrls: array (nullable = true)\n",
      " |    |-- element: string (containsNull = true)\n",
      " |-- pmid: string (nullable = true)\n",
      " |-- s2PdfUrl: string (nullable = true)\n",
      " |-- s2Url: string (nullable = true)\n",
      " |-- sources: array (nullable = true)\n",
      " |    |-- element: string (containsNull = true)\n",
      " |-- title: string (nullable = true)\n",
      " |-- venue: string (nullable = true)\n",
      " |-- year: long (nullable = true)\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "SS = [authors: array<struct<ids:array<string>,name:string>>, doi: string ... 19 more fields]\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "source": "user"
    },
    {
     "data": {
      "text/plain": [
       "[authors: array<struct<ids:array<string>,name:string>>, doi: string ... 19 more fields]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val SS = spark.read.json(\"/dataRCSC/SS-2020-05-27/*.gz\")\n",
    "SS.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+--------+---+--------+-----+----+-----+---------+-------+--------+------------+-------+-------+------+----------------+--------+--------------+--------------+---+-----+----------+--------------+----------+--------------+----+-----------+------+---------+------------+--------------+---------------------+----------------+-------+--------+------+----+----------+----------+-------------+--------------+-------------+-----------+------------+-----+----+-------+\n",
      "|summary|cord_uid|sha|source_x|title| doi|pmcid|pubmed_id|license|abstract|publish_time|authors|journal|mag_id|who_covidence_id|arxiv_id|pdf_json_files|pmc_json_files|url|s2_id|pmcAuthors|pmcAffiliation|pdfAuthors|pdfAffiliation|pmid|annotations|MAGids|authorids|authorOrders|affiliationids|affiliationNormalized|affiliationNames|GRIDids|xref_doi|art_no| UID|WoSauthors|WoSjournal|score_title_0|score_author_0|ID_researcher|Author_name|author_order|SSdoi|SSid|SSmagId|\n",
      "+-------+--------+---+--------+-----+----+-----+---------+-------+--------+------------+-------+-------+------+----------------+--------+--------------+--------------+---+-----+----------+--------------+----------+--------------+----+-----------+------+---------+------------+--------------+---------------------+----------------+-------+--------+------+----+----------+----------+-------------+--------------+-------------+-----------+------------+-----+----+-------+\n",
      "|  count|    5863|415|    5863| 5863|3172|  559|      246|   5863|    3431|        5859|   5863|   5659|     0|            5272|       3|           415|           299|597| 5556|       299|           299|       401|           401|5528|       1641|  3220|     3220|        3220|          3220|                 3220|            3220|   3112|    1889|   479|5863|      5863|      5863|         5863|          5863|         5863|       5863|        5863|   70|  70|     70|\n",
      "+-------+--------+---+--------+-----+----+-----+---------+-------+--------+------------+-------+-------+------+----------------+--------+--------------+--------------+---+-----+----------+--------------+----------+--------------+----+-----------+------+---------+------------+--------------+---------------------+----------------+-------+--------+------+----+----------+----------+-------------+--------------+-------------+-----------+------------+-----+----+-------+\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "SS0 = [SSauthorNames: array<string>, SSauthorIDs: array<array<string>> ... 4 more fields]\n",
       "meta1 = [cord_uid: string, sha: string ... 46 more fields]\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "source": "user"
    },
    {
     "data": {
      "text/plain": [
       "[cord_uid: string, sha: string ... 46 more fields]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val SS0 = SS.select(\"authors.name\", \"authors.ids\", \"doi\", \"id\", \"inCitations\", \"magId\")\n",
    "            .toDF(\"SSauthorNames\",\"SSauthorIDs\",\"SSdoi\",\"SSid\",\"SSrefs\",\"SSmagId\").cache()\n",
    "val meta1 = meta0.join(SS0, meta0(\"sha\")===SS0(\"SSid\"), \"left\")\n",
    "meta1.describe().filter($\"summary\" === \"count\").show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------+-------------------+----+\n",
      "|        col|size(SSauthorNames)|size|\n",
      "+-----------+-------------------+----+\n",
      "| [51978410]|                  1|   1|\n",
      "| [71630645]|                  3|   1|\n",
      "| [52393267]|                  3|   1|\n",
      "| [72050485]|                  3|   1|\n",
      "|  [1776025]|                  6|   1|\n",
      "|  [2836018]|                  6|   1|\n",
      "| [47422037]|                  6|   1|\n",
      "|  [1679387]|                  6|   1|\n",
      "|[145799243]|                  6|   1|\n",
      "|[145754722]|                  6|   1|\n",
      "|[114402012]|                  1|   1|\n",
      "|[136282636]|                  1|   1|\n",
      "|[146709087]|                  3|   1|\n",
      "| [51218054]|                  3|   1|\n",
      "|[144681826]|                  3|   1|\n",
      "|[120457898]|                  1|   1|\n",
      "|[152634023]|                  4|   1|\n",
      "|[153204165]|                  4|   1|\n",
      "|  [4244638]|                  4|   1|\n",
      "|  [1796371]|                  4|   1|\n",
      "+-----------+-------------------+----+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "SSexp = [col: array<string>, size(SSauthorNames): int ... 1 more field]\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "source": "user"
    },
    {
     "data": {
      "text/plain": [
       "[col: array<string>, size(SSauthorNames): int ... 1 more field]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val SSexp = SS0.select(explode($\"SSauthorIDs\"),size($\"SSauthorNames\")).withColumn(\"size\", size($\"col\"))\n",
    "SSexp.orderBy($\"size\".desc).show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+--------+---+--------+-----+----+-----+---------+-------+--------+------------+-------+-------+------+----------------+--------+--------------+--------------+---+-----+----------+--------------+----------+--------------+----+-----------+------+---------+------------+--------------+---------------------+----------------+-------+--------+------+----+----------+----------+-------------+--------------+-------------+-----------+------------+-----+----+-------+\n",
      "|summary|cord_uid|sha|source_x|title| doi|pmcid|pubmed_id|license|abstract|publish_time|authors|journal|mag_id|who_covidence_id|arxiv_id|pdf_json_files|pmc_json_files|url|s2_id|pmcAuthors|pmcAffiliation|pdfAuthors|pdfAffiliation|pmid|annotations|MAGids|authorids|authorOrders|affiliationids|affiliationNormalized|affiliationNames|GRIDids|xref_doi|art_no| UID|WoSauthors|WoSjournal|score_title_0|score_author_0|ID_researcher|Author_name|author_order|SSdoi|SSid|SSmagId|\n",
      "+-------+--------+---+--------+-----+----+-----+---------+-------+--------+------------+-------+-------+------+----------------+--------+--------------+--------------+---+-----+----------+--------------+----------+--------------+----+-----------+------+---------+------------+--------------+---------------------+----------------+-------+--------+------+----+----------+----------+-------------+--------------+-------------+-----------+------------+-----+----+-------+\n",
      "|  count|    5934|416|    5934| 5934|3243|  560|      248|   5934|    3486|        5930|   5934|   5729|     0|            5340|       3|           416|           300|600| 5626|       300|           300|       402|           402|5599|       1655|  3261|     3261|        3261|          3261|                 3261|            3261|   3153|    1889|   479|5934|      5934|      5934|         5934|          5934|         5934|       5934|        5934| 3241|3241|   3241|\n",
      "+-------+--------+---+--------+-----+----+-----+---------+-------+--------+------------+-------+-------+------+----------------+--------+--------------+--------------+---+-----+----------+--------------+----------+--------------+----+-----------+------+---------+------------+--------------+---------------------+----------------+-------+--------+------+----+----------+----------+-------------+--------------+-------------+-----------+------------+-----+----+-------+\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "meta2 = [cord_uid: string, sha: string ... 46 more fields]\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "source": "user"
    },
    {
     "data": {
      "text/plain": [
       "[cord_uid: string, sha: string ... 46 more fields]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val meta2 = meta0.join(SS0, meta0(\"doi\")===SS0(\"SSdoi\"), \"left\").cache()\n",
    "meta2.describe().filter($\"summary\" === \"count\").show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+--------------------+--------------------+--------------------+--------------------+\n",
      "|cord_uid|              SSdois|    SSauthorNameList|      SSauthorIDList|           SSrefList|\n",
      "+--------+--------------------+--------------------+--------------------+--------------------+\n",
      "|iqpgmls6|10.1016/j.bbmt.20...|Sakara Hutspardol...|12492852;49771725...|caf95fa590e7cbb34...|\n",
      "|xe02b8ga|10.1016/j.vaccine...|Markus Czub;Hana ...|4789603;4802727;5...|15b0a02a3a59cdc50...|\n",
      "|6b35vzd8|10.1016/j.ajic.20...|David M Moore;Bru...|47932570;12068284...|cb8ad6cfaa028e356...|\n",
      "|n7uzdsx7|10.1016/j.ajic.20...|David M Moore;Bru...|47932570;12068284...|cb8ad6cfaa028e356...|\n",
      "|xavtxl4m|10.2196/jmir.7.4.e46|Carlos Acosta Riz...|144324018;4811408...|c64ba547ae3ac0b53...|\n",
      "|xolw2lti|10.2196/jmir.7.4.e46|Carlos Acosta Riz...|144324018;4811408...|c64ba547ae3ac0b53...|\n",
      "|7v1ng0ne|10.1146/annurev.m...|Danuta M. Skowron...|6135438;5252414;3...|7b4d551e70ca2ddbb...|\n",
      "|xjt86q94|10.1146/annurev.m...|Danuta M. Skowron...|6135438;5252414;3...|7b4d551e70ca2ddbb...|\n",
      "|3jd27e11|                    |                null|                null|                null|\n",
      "|7b4ssyzm|                    |                null|                null|                null|\n",
      "|a45bspd2|                    |                null|                null|                null|\n",
      "|dn1nse7h|                    |                null|                null|                null|\n",
      "|dwug2azi|                    |                null|                null|                null|\n",
      "|rw3p6wle|                    |                null|                null|                null|\n",
      "|uhpmh2ah|                    |                null|                null|                null|\n",
      "|uq9nutz9|                    |                null|                null|                null|\n",
      "|z6trtz8s|                    |                null|                null|                null|\n",
      "|zgltws0c|                    |                null|                null|                null|\n",
      "|lwvlajry|   10.1159/000087959|   Timothy Caulfield|           145582198|e9ebf20b83c95f7db...|\n",
      "|z00zvdxo|   10.1159/000087959|   Timothy Caulfield|           145582198|e9ebf20b83c95f7db...|\n",
      "+--------+--------------------+--------------------+--------------------+--------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "SSmatched2.show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+---+------+\n",
      "|summary|UID|SSdois|\n",
      "+-------+---+------+\n",
      "|  count|315|   315|\n",
      "+-------+---+------+\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "SSmatched = [UID: string, SSdois: string ... 3 more fields]\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "source": "user"
    },
    {
     "data": {
      "text/plain": [
       "[UID: string, SSdois: string ... 3 more fields]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val SSmatched = meta2.groupBy(\"doi\").agg(concat_ws(\";\", collect_list(\"SSdoi\")) as \"SSdois\",\n",
    "                                            first(\"SSauthorNames\") as \"SSauthorNames\",\n",
    "                                            first(\"SSauthorIDs\") as \"SSauthorIDs\",\n",
    "                                            first(\"SSrefs\") as \"SSrefs\")\n",
    "\n",
    "SSmatched.describe().filter($\"summary\" === \"count\").show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+--------+------+\n",
      "|summary|cord_uid|SSdois|\n",
      "+-------+--------+------+\n",
      "|  count|    4989|  4989|\n",
      "+-------+--------+------+\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "SSmatched = [cord_uid: string, SSdois: string ... 3 more fields]\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "source": "user"
    },
    {
     "data": {
      "text/plain": [
       "[cord_uid: string, SSdois: string ... 3 more fields]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val SSmatched = meta2.groupBy(\"cord_uid\").agg(concat_ws(\";\", collect_list(\"SSdoi\")) as \"SSdois\",\n",
    "                                            first(\"SSauthorNames\") as \"SSauthorNames\",\n",
    "                                            first(\"SSauthorIDs\") as \"SSauthorIDs\",\n",
    "                                            first(\"SSrefs\") as \"SSrefs\")\n",
    "\n",
    "SSmatched.describe().filter($\"summary\" === \"count\").show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+--------+------+----------------+--------------+---------+\n",
      "|summary|cord_uid|SSdois|SSauthorNameList|SSauthorIDList|SSrefList|\n",
      "+-------+--------+------+----------------+--------------+---------+\n",
      "|  count|    4989|  4989|            2727|          2727|     2727|\n",
      "+-------+--------+------+----------------+--------------+---------+\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "SSmatched2 = [cord_uid: string, SSdois: string ... 3 more fields]\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "source": "user"
    },
    {
     "data": {
      "text/plain": [
       "lastException: Throwable = null\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "source": "user"
    },
    {
     "data": {
      "text/plain": [
       "[cord_uid: string, SSdois: string ... 3 more fields]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val SSmatched2 = SSmatched.withColumn(\"SSauthorNameList\", array_join($\"SSauthorNames\", \";\"))\n",
    "                        .withColumn(\"SSauthorIDList\", array_join(flatten($\"SSauthorIDs\"), \";\"))\n",
    "                        .withColumn(\"SSrefList\", array_join($\"SSrefs\", \";\"))\n",
    "                        .select(\"cord_uid\",\"SSdois\",\"SSauthorNameList\",\"SSauthorIDList\",\"SSrefList\")\n",
    "SSmatched2.describe().filter($\"summary\" === \"count\").show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Name: Unknown Error\n",
       "Message: lastException: Throwable = null\n",
       "<console>:46: error: not found: value SSmatched2\n",
       "       val meta2 = meta0.join(SSmatched2, Seq(\"UID\"), \"left\").cache()\n",
       "                              ^\n",
       "\n",
       "StackTrace: "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val meta2 = meta0.join(SSmatched2, Seq(\"UID\"), \"left\").cache()\n",
    "meta2.describe().filter($\"summary\" === \"count\").show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "meta2.drop(\"_c0\",\"SSAuthors\",\"SSRefs\")\n",
    "    .coalesce(1).write.mode(SaveMode.Overwrite)\n",
    "    .format(\"com.databricks.spark.csv\")\n",
    "    .option(\"header\", \"true\")\n",
    "    .option(\"quote\", \"\\\"\")\n",
    "    .save(\"wasbs://mag-2020-05-29@iunimag.blob.core.windows.net/2020-06-08/metaAuthors.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"abstract\":\"ObjectiveThis retrospective chart review describes the epidemiology and clinical features of 40 patients with culture-proven Mycoplasma pneumoniae infections at King Abdulaziz University Hospital, Jeddah, Saudi Arabia.MethodsPatients with positive M. pneumoniae cultures from respiratory specimens from January 1997 through December 1998 were identified through the Microbiology records. Charts of patients were reviewed.Results40 patients were identified, 33 (82.5%) of whom required admission. Most infections (92.5%) were community-acquired. The infection affected all age groups but was most common in infants (32.5%) and pre-school children (22.5%). It occurred year-round but was most common in the fall (35%) and spring (30%). More than three-quarters of patients (77.5%) had comorbidities. Twenty-four isolates (60%) were associated with pneumonia, 14 (35%) with upper respiratory tract infections, and 2 (5%) with bronchiolitis. Cough (82.5%), fever (75%), and malaise (58.8%) were the most common symptoms, and crepitations (60%), and wheezes (40%) were the most common signs. Most patients with pneumonia had crepitations (79.2%) but only 25% had bronchial breathing. Immunocompromised patients were more likely than non-immunocompromised patients to present with pneumonia (8/9 versus 16/31, P = 0.05). Of the 24 patients with pneumonia, 14 (58.3%) had uneventful recovery, 4 (16.7%) recovered following some complications, 3 (12.5%) died because of M pneumoniae infection, and 3 (12.5%) died due to underlying comorbidities. The 3 patients who died of M pneumoniae pneumonia had other comorbidities.Conclusionour results were similar to published data except for the finding that infections were more common in infants and preschool children and that the mortality rate of pneumonia in patients with comorbidities was high.\",\"arxivId\":null,\"authors\":[{\"authorId\":\"2843307\",\"name\":\"Tariq A Madani\",\"url\":\"https://www.semanticscholar.org/author/2843307\"},{\"authorId\":\"7699774\",\"name\":\"Aisha A. Al-Ghamdi\",\"url\":\"https://www.semanticscholar.org/author/7699774\"}],\"citationVelocity\":0,\"citations\":[{\"arxivId\":null,\"authors\":[{\"authorId\":\"101823415\",\"name\":\"Osama Mohammed Saed Abdul-Wahab\",\"url\":\"https://www.semanticscholar.org/author/101823415\"},{\"authorId\":\"1401025118\",\"name\":\"Ahmed M. Al-Hakami\",\"url\":\"https://www.semanticscholar.org/author/1401025118\"},{\"authorId\":\"22699901\",\"name\":\"Ayed Abdullah Shati\",\"url\":\"https://www.semanticscholar.org/author/22699901\"},{\"authorId\":\"5113040\",\"name\":\"Ali Mohammed Alsuheel\",\"url\":\"https://www.semanticscholar.org/author/5113040\"},{\"authorId\":\"47311092\",\"name\":\"Ashish Kumar\",\"url\":\"https://www.semanticscholar.org/author/47311092\"},{\"authorId\":\"74001869\",\"name\":\"Fateha Benahmed\",\"url\":\"https://www.semanticscholar.org/author/74001869\"}],\"doi\":\"10.3844/ajisp.2013.1.8\",\"intent\":[],\"isInfluential\":false,\"paperId\":\"b15358c6fb33a9dd2674baa55924d7eb41faad2e\",\"title\":\"ASSOCIATION OF MYCOPLASMA PNEUMONIAE WITH RESPIRATORY TRACT INFECTIONS IN CHILDREN\",\"url\":\"https://www.semanticscholar.org/paper/b15358c6fb33a9dd2674baa55924d7eb41faad2e\",\"venue\":\"\",\"year\":2013},{\"arxivId\":null,\"authors\":[{\"authorId\":\"80896877\",\"name\":\"Raef Ahmed Muhammad Qutub\",\"url\":\"https://www.semanticscholar.org/author/80896877\"},{\"authorId\":\"12063487\",\"name\":\"Sari Ibrahim Asiri\",\"url\":\"https://www.semanticscholar.org/author/12063487\"},{\"authorId\":\"80947657\",\"name\":\"Eatetal Abdurabuh\",\"url\":\"https://www.semanticscholar.org/author/80947657\"},{\"authorId\":\"51373349\",\"name\":\"Alaa Ahmed Bugis\",\"url\":\"https://www.semanticscholar.org/author/51373349\"},{\"authorId\":\"80502368\",\"name\":\"Kholoud Alsemri\",\"url\":\"https://www.semanticscholar.org/author/80502368\"}],\"doi\":\"10.5455/ijmsph.2013.140320131\",\"intent\":[\"background\"],\"isInfluential\":false,\"paperId\":\"9535a3f08f57a5c1450557506167de73f66087b6\",\"title\":\"Confusion in a patient with mycoplasma pneumonia lowers threshhold to lumbar puncture despite hyponatremia\",\"url\":\"https://www.semanticscholar.org/paper/9535a3f08f57a5c1450557506167de73f66087b6\",\"venue\":\"\",\"year\":2013},{\"arxivId\":null,\"authors\":[{\"authorId\":\"120086033\",\"name\":\"Jin Soo Kim\",\"url\":\"https://www.semanticscholar.org/author/120086033\"},{\"authorId\":\"47778509\",\"name\":\"Jeong Hee Ko\",\"url\":\"https://www.semanticscholar.org/author/47778509\"},{\"authorId\":\"4580041\",\"name\":\"Sung Hee Oh\",\"url\":\"https://www.semanticscholar.org/author/4580041\"}],\"doi\":\"10.14776/kjpid.2013.20.2.71\",\"intent\":[],\"isInfluential\":false,\"paperId\":\"3c1588b7d42fa7809ebdf6294460898e323237de\",\"title\":\"Indirect Particle Agglutination Antibody Testing for Early Diagnosis of Mycoplasma pneumoniae pneumonia in Children\",\"url\":\"https://www.semanticscholar.org/paper/3c1588b7d42fa7809ebdf6294460898e323237de\",\"venue\":\"\",\"year\":2013},{\"arxivId\":null,\"authors\":[{\"authorId\":\"108205515\",\"name\":\"Achraf Zouari\",\"url\":\"https://www.semanticscholar.org/author/108205515\"},{\"authorId\":\"145953499\",\"name\":\"A. Touati\",\"url\":\"https://www.semanticscholar.org/author/145953499\"},{\"authorId\":\"153692019\",\"name\":\"Hanen Smaoui\",\"url\":\"https://www.semanticscholar.org/author/153692019\"},{\"authorId\":\"102030741\",\"name\":\"Didier Brun\",\"url\":\"https://www.semanticscholar.org/author/102030741\"},{\"authorId\":\"1398691624\",\"name\":\"K. Kasdaghli\",\"url\":\"https://www.semanticscholar.org/author/1398691624\"},{\"authorId\":\"5694347\",\"name\":\"Khaled Menif\",\"url\":\"https://www.semanticscholar.org/author/5694347\"},{\"authorId\":\"115465853\",\"name\":\"Nejla Ben Jaballah\",\"url\":\"https://www.semanticscholar.org/author/115465853\"},{\"authorId\":\"47512031\",\"name\":\"E Ben Hassen\",\"url\":\"https://www.semanticscholar.org/author/47512031\"},{\"authorId\":\"48463699\",\"name\":\"Nicole Guiso\",\"url\":\"https://www.semanticscholar.org/author/48463699\"},{\"authorId\":\"6553607\",\"name\":\"Amel K\\u00e9chrid\",\"url\":\"https://www.semanticscholar.org/author/6553607\"}],\"doi\":\"10.1007/s15010-011-0179-4\",\"intent\":[\"background\"],\"isInfluential\":false,\"paperId\":\"7796661eb911fd8ffb8a767ba8fc5bffb127a701\",\"title\":\"Dual infection with Bordetella pertussis and Mycoplasma pneumoniae in three infants: case reports\",\"url\":\"https://www.semanticscholar.org/paper/7796661eb911fd8ffb8a767ba8fc5bffb127a701\",\"venue\":\"Infection\",\"year\":2011},{\"arxivId\":null,\"authors\":[{\"authorId\":\"79022650\",\"name\":\"Ali H. Bayati\",\"url\":\"https://www.semanticscholar.org/author/79022650\"},{\"authorId\":\"82152837\",\"name\":\"Pakhshan M. Faraj\",\"url\":\"https://www.semanticscholar.org/author/82152837\"},{\"authorId\":\"66787442\",\"name\":\"Khalid Hama Salih\",\"url\":\"https://www.semanticscholar.org/author/66787442\"}],\"doi\":null,\"intent\":[],\"isInfluential\":false,\"paperId\":\"e9f834068d3b94a201b4f2fee77604dbf6ad0382\",\"title\":\"Epidemiological, Clinical, and immunological characteristics of Mycoplasma pneumonia infections among a group of hospitalized children in Suleimani city/Iraq\",\"url\":\"https://www.semanticscholar.org/paper/e9f834068d3b94a201b4f2fee77604dbf6ad0382\",\"venue\":\"\",\"year\":2014},{\"arxivId\":null,\"authors\":[{\"authorId\":\"48047756\",\"name\":\"J P de Boer\",\"url\":\"https://www.semanticscholar.org/author/48047756\"},{\"authorId\":\"4653923\",\"name\":\"S J A Aerdts\",\"url\":\"https://www.semanticscholar.org/author/4653923\"},{\"authorId\":\"7202207\",\"name\":\"Paul H. P. Groeneveld\",\"url\":\"https://www.semanticscholar.org/author/7202207\"}],\"doi\":null,\"intent\":[],\"isInfluential\":false,\"paperId\":\"71dac6dfdd2cac4cfeae00f2936b9bb901237d1b\",\"title\":\"Severe diffuse interstitial pneumonia due to Mycoplasma pneumoniae in a patient with respiratory insufficiency.\",\"url\":\"https://www.semanticscholar.org/paper/71dac6dfdd2cac4cfeae00f2936b9bb901237d1b\",\"venue\":\"The Netherlands journal of medicine\",\"year\":2003},{\"arxivId\":null,\"authors\":[{\"authorId\":\"2313459\",\"name\":\"Abraham Tareq Yacoub\",\"url\":\"https://www.semanticscholar.org/author/2313459\"},{\"authorId\":\"84417053\",\"name\":\"Sarah I. Khwaja\",\"url\":\"https://www.semanticscholar.org/author/84417053\"},{\"authorId\":\"153414509\",\"name\":\"Talabi Taiwo\",\"url\":\"https://www.semanticscholar.org/author/153414509\"},{\"authorId\":\"5728033\",\"name\":\"Ju Hee Kim\",\"url\":\"https://www.semanticscholar.org/author/5728033\"},{\"authorId\":\"3992947\",\"name\":\"Sowmya Nanjappa\",\"url\":\"https://www.semanticscholar.org/author/3992947\"},{\"authorId\":\"2604531\",\"name\":\"John N Greene\",\"url\":\"https://www.semanticscholar.org/author/2604531\"}],\"doi\":\"10.1097/ipc.0000000000000392\",\"intent\":[\"background\"],\"isInfluential\":false,\"paperId\":\"a27cd56c005715c1bf0981bc61dd0a422cf76025\",\"title\":\"Mycoplasma pneumoniae in the Immunocompromised Host: A Case Series and Review of Literature\",\"url\":\"https://www.semanticscholar.org/paper/a27cd56c005715c1bf0981bc61dd0a422cf76025\",\"venue\":\"\",\"year\":2016},{\"arxivId\":null,\"authors\":[{\"authorId\":\"46794877\",\"name\":\"Sharlene Govender\",\"url\":\"https://www.semanticscholar.org/author/46794877\"}],\"doi\":null,\"intent\":[\"background\"],\"isInfluential\":false,\"paperId\":\"7658ff262455a2bd6efdabd07670c3a48c35adfb\",\"title\":\"Epidemiology and antibiotic susceptibility patterns of mycoplasma sp. and ureaplasma urealyticum\",\"url\":\"https://www.semanticscholar.org/paper/7658ff262455a2bd6efdabd07670c3a48c35adfb\",\"venue\":\"\",\"year\":2010}],\"corpusId\":20142301,\"doi\":\"10.1186/1471-2334-1-6\",\"fieldsOfStudy\":[\"Medicine\"],\"influentialCitationCount\":0,\"is_open_access\":true,\"is_publisher_licensed\":true,\"paperId\":\"dee1269e42a568e8ab3afa5317c45ae4007ebe81\",\"references\":[{\"arxivId\":null,\"authors\":[{\"authorId\":\"46456870\",\"name\":\"James D. Cherry\",\"url\":\"https://www.semanticscholar.org/author/46456870\"}],\"doi\":\"10.1093/clinids/17.Supplement_1.S47\",\"intent\":[\"background\"],\"isInfluential\":false,\"paperId\":\"61c0cc81ad4dfce72db60ea1079d4c28449c2d3a\",\"title\":\"Anemia and mucocutaneous lesions due to Mycoplasma pneumoniae infections.\",\"url\":\"https://www.semanticscholar.org/paper/61c0cc81ad4dfce72db60ea1079d4c28449c2d3a\",\"venue\":\"Clinical infectious diseases : an official publication of the Infectious Diseases Society of America\",\"year\":1993},{\"arxivId\":null,\"authors\":[{\"authorId\":\"145230033\",\"name\":\"T J Marrie\",\"url\":\"https://www.semanticscholar.org/author/145230033\"},{\"authorId\":\"145430571\",\"name\":\"Rosanna W Peeling\",\"url\":\"https://www.semanticscholar.org/author/145430571\"},{\"authorId\":\"144676704\",\"name\":\"Michael J. Fine\",\"url\":\"https://www.semanticscholar.org/author/144676704\"},{\"authorId\":\"1931211\",\"name\":\"Daniel Singer\",\"url\":\"https://www.semanticscholar.org/author/1931211\"},{\"authorId\":\"2242199\",\"name\":\"Christopher M. Coley\",\"url\":\"https://www.semanticscholar.org/author/2242199\"},{\"authorId\":\"1796768\",\"name\":\"Wishwa Kapoor\",\"url\":\"https://www.semanticscholar.org/author/1796768\"}],\"doi\":\"10.1016/S0002-9343(96)00255-0\",\"intent\":[\"background\"],\"isInfluential\":false,\"paperId\":\"fb2241c270fd5d0b71acc68a64265de14dc062b7\",\"title\":\"Ambulatory patients with community-acquired pneumonia: the frequency of atypical agents and clinical course.\",\"url\":\"https://www.semanticscholar.org/paper/fb2241c270fd5d0b71acc68a64265de14dc062b7\",\"venue\":\"The American journal of medicine\",\"year\":1996},{\"arxivId\":null,\"authors\":[{\"authorId\":\"152323845\",\"name\":\"Nabil Yasin Al Kurashi\",\"url\":\"https://www.semanticscholar.org/author/152323845\"},{\"authorId\":\"1414033366\",\"name\":\"A. Al-Hamdan\",\"url\":\"https://www.semanticscholar.org/author/1414033366\"},{\"authorId\":\"144317170\",\"name\":\"Ezzeldin M. Ibrahim\",\"url\":\"https://www.semanticscholar.org/author/144317170\"},{\"authorId\":\"13167305\",\"name\":\"Hassan Y. Al-Idrissi\",\"url\":\"https://www.semanticscholar.org/author/13167305\"},{\"authorId\":\"1422543875\",\"name\":\"Talal H Al-Bayari\",\"url\":\"https://www.semanticscholar.org/author/1422543875\"}],\"doi\":\"10.1136/thx.47.2.115\",\"intent\":[],\"isInfluential\":false,\"paperId\":\"f9f9321da9e109e416296bf88d0aa58e2d42d90a\",\"title\":\"Community acquired acute bacterial and atypical pneumonia in Saudi Arabia.\",\"url\":\"https://www.semanticscholar.org/paper/f9f9321da9e109e416296bf88d0aa58e2d42d90a\",\"venue\":\"Thorax\",\"year\":1992},{\"arxivId\":null,\"authors\":[{\"authorId\":\"11573662\",\"name\":\"William J. Mogabgab\",\"url\":\"https://www.semanticscholar.org/author/11573662\"}],\"doi\":\"10.1164/arrd.1968.97.3.345\",\"intent\":[\"background\"],\"isInfluential\":false,\"paperId\":\"812bb7f5cb8e064c0e174ca9fa821201eb830fe0\",\"title\":\"Mycoplasma pneumoniae and adenovirus respiratory illnesses in military and university personnel, 1959-1966.\",\"url\":\"https://www.semanticscholar.org/paper/812bb7f5cb8e064c0e174ca9fa821201eb830fe0\",\"venue\":\"The American review of respiratory disease\",\"year\":1968},{\"arxivId\":null,\"authors\":[{\"authorId\":\"47970408\",\"name\":\"Richard T. Kenney\",\"url\":\"https://www.semanticscholar.org/author/47970408\"},{\"authorId\":\"47785867\",\"name\":\"Jennifer S. Li\",\"url\":\"https://www.semanticscholar.org/author/47785867\"},{\"authorId\":\"82754722\",\"name\":\"W A Clyde\",\"url\":\"https://www.semanticscholar.org/author/82754722\"},{\"authorId\":\"5760755\",\"name\":\"Thomas Carl Wall\",\"url\":\"https://www.semanticscholar.org/author/5760755\"},{\"authorId\":\"1429572226\",\"name\":\"Christopher O'Connor\",\"url\":\"https://www.semanticscholar.org/author/1429572226\"},{\"authorId\":\"49353623\",\"name\":\"Paul T Campbell\",\"url\":\"https://www.semanticscholar.org/author/49353623\"},{\"authorId\":\"10199919\",\"name\":\"P Van Trigt\",\"url\":\"https://www.semanticscholar.org/author/10199919\"},{\"authorId\":\"5176584\",\"name\":\"G. Ralph Corey\",\"url\":\"https://www.semanticscholar.org/author/5176584\"}],\"doi\":\"10.1093/clinids/17.Supplement_1.S58\",\"intent\":[\"background\"],\"isInfluential\":false,\"paperId\":\"678e7949f502c4012e9bc118631bfdaca6c839e5\",\"title\":\"Mycoplasmal pericarditis: evidence of invasive disease.\",\"url\":\"https://www.semanticscholar.org/paper/678e7949f502c4012e9bc118631bfdaca6c839e5\",\"venue\":\"Clinical infectious diseases : an official publication of the Infectious Diseases Society of America\",\"year\":1993},{\"arxivId\":null,\"authors\":[{\"authorId\":\"144053415\",\"name\":\"Devora Lieberman\",\"url\":\"https://www.semanticscholar.org/author/144053415\"},{\"authorId\":\"4548853\",\"name\":\"Francisc Schlaeffer\",\"url\":\"https://www.semanticscholar.org/author/4548853\"},{\"authorId\":\"144053415\",\"name\":\"Devora Lieberman\",\"url\":\"https://www.semanticscholar.org/author/144053415\"},{\"authorId\":\"66384634\",\"name\":\"Sheryl Horowitz\",\"url\":\"https://www.semanticscholar.org/author/66384634\"},{\"authorId\":\"2061007\",\"name\":\"Ossi Horovitz\",\"url\":\"https://www.semanticscholar.org/author/2061007\"},{\"authorId\":\"2582625\",\"name\":\"Avi Porath\",\"url\":\"https://www.semanticscholar.org/author/2582625\"}],\"doi\":\"10.1159/000196557\",\"intent\":[\"background\"],\"isInfluential\":false,\"paperId\":\"daf3404ecf25c31676c784c25d973de282eb6275\",\"title\":\"Mycoplasma pneumoniae community-acquired pneumonia: a review of 101 hospitalized adult patients.\",\"url\":\"https://www.semanticscholar.org/paper/daf3404ecf25c31676c784c25d973de282eb6275\",\"venue\":\"Respiration; international review of thoracic diseases\",\"year\":1996},{\"arxivId\":null,\"authors\":[{\"authorId\":\"6716646\",\"name\":\"Gary Maartens\",\"url\":\"https://www.semanticscholar.org/author/6716646\"},{\"authorId\":\"51096509\",\"name\":\"Stephanie J. Lewis\",\"url\":\"https://www.semanticscholar.org/author/51096509\"},{\"authorId\":\"15119707\",\"name\":\"C de Goveia\",\"url\":\"https://www.semanticscholar.org/author/15119707\"},{\"authorId\":\"49131111\",\"name\":\"Catheleen Bartie\",\"url\":\"https://www.semanticscholar.org/author/49131111\"},{\"authorId\":\"5641157\",\"name\":\"Denise Roditi\",\"url\":\"https://www.semanticscholar.org/author/5641157\"},{\"authorId\":\"2187399\",\"name\":\"Keith P. Klugman\",\"url\":\"https://www.semanticscholar.org/author/2187399\"}],\"doi\":null,\"intent\":[\"background\"],\"isInfluential\":false,\"paperId\":\"32b9daff0f4921c629fb1ef40ab7936a290bc10b\",\"title\":\"'Atypical' bacteria are a common cause of community-acquired pneumonia in hospitalised adults.\",\"url\":\"https://www.semanticscholar.org/paper/32b9daff0f4921c629fb1ef40ab7936a290bc10b\",\"venue\":\"South African medical journal = Suid-Afrikaanse tydskrif vir geneeskunde\",\"year\":1994},{\"arxivId\":null,\"authors\":[{\"authorId\":\"48556739\",\"name\":\"Hjordis M. Foy\",\"url\":\"https://www.semanticscholar.org/author/48556739\"},{\"authorId\":\"74329917\",\"name\":\"George E. Kenny\",\"url\":\"https://www.semanticscholar.org/author/74329917\"},{\"authorId\":\"3995454\",\"name\":\"Marion K. Cooney\",\"url\":\"https://www.semanticscholar.org/author/3995454\"},{\"authorId\":\"145580919\",\"name\":\"I. Dal Allan\",\"url\":\"https://www.semanticscholar.org/author/145580919\"}],\"doi\":\"10.1093/infdis/139.6.681\",\"intent\":[\"background\",\"result\"],\"isInfluential\":true,\"paperId\":\"ed0c66a4eb3810006d276dbfbf70debfc26747f0\",\"title\":\"Long-term epidemiology of infections with Mycoplasma pneumoniae.\",\"url\":\"https://www.semanticscholar.org/paper/ed0c66a4eb3810006d276dbfbf70debfc26747f0\",\"venue\":\"The Journal of infectious diseases\",\"year\":1979},{\"arxivId\":null,\"authors\":[{\"authorId\":\"113947573\",\"name\":\"A. Raouf Mohamed\",\"url\":\"https://www.semanticscholar.org/author/113947573\"},{\"authorId\":\"46626279\",\"name\":\"D. A. Evans\",\"url\":\"https://www.semanticscholar.org/author/46626279\"}],\"doi\":\"10.1016/S0163-4453(87)90756-0\",\"intent\":[],\"isInfluential\":false,\"paperId\":\"8e13f34b715d413c3b1a8cb451200a910a38891e\",\"title\":\"The spectrum of pneumonia in 1983 at the Riyadh Armed Forces Hospital.\",\"url\":\"https://www.semanticscholar.org/paper/8e13f34b715d413c3b1a8cb451200a910a38891e\",\"venue\":\"The Journal of infection\",\"year\":1987},{\"arxivId\":null,\"authors\":[{\"authorId\":\"145230033\",\"name\":\"T J Marrie\",\"url\":\"https://www.semanticscholar.org/author/145230033\"}],\"doi\":\"10.1001/archinte.1993.00410040054008\",\"intent\":[\"background\"],\"isInfluential\":false,\"paperId\":\"02cf12a4ea37721ad4b7944a5d06b5d1f7e59f13\",\"title\":\"Mycoplasma pneumoniae pneumonia requiring hospitalization, with emphasis on infection in the elderly.\",\"url\":\"https://www.semanticscholar.org/paper/02cf12a4ea37721ad4b7944a5d06b5d1f7e59f13\",\"venue\":\"Archives of internal medicine\",\"year\":1993},{\"arxivId\":null,\"authors\":[{\"authorId\":\"48556739\",\"name\":\"Hjordis M. Foy\",\"url\":\"https://www.semanticscholar.org/author/48556739\"}],\"doi\":\"10.1093/clinids/17.Supplement_1.S37\",\"intent\":[],\"isInfluential\":false,\"paperId\":\"6edb2d9f141c997dc2212359efe3782e7f97e18a\",\"title\":\"Infections caused by Mycoplasma pneumoniae and possible carrier state in different populations of patients.\",\"url\":\"https://www.semanticscholar.org/paper/6edb2d9f141c997dc2212359efe3782e7f97e18a\",\"venue\":\"Clinical infectious diseases : an official publication of the Infectious Diseases Society of America\",\"year\":1993},{\"arxivId\":null,\"authors\":[{\"authorId\":\"33129938\",\"name\":\"Nayeem Ali\",\"url\":\"https://www.semanticscholar.org/author/33129938\"},{\"authorId\":\"6708490\",\"name\":\"Margaret Sillis\",\"url\":\"https://www.semanticscholar.org/author/6708490\"},{\"authorId\":\"51343536\",\"name\":\"Basil E. Andrews\",\"url\":\"https://www.semanticscholar.org/author/51343536\"},{\"authorId\":\"6368704\",\"name\":\"Patricia F. Jenkins\",\"url\":\"https://www.semanticscholar.org/author/6368704\"},{\"authorId\":\"145197354\",\"name\":\"Brian D W Harrison\",\"url\":\"https://www.semanticscholar.org/author/145197354\"}],\"doi\":\"10.1093/oxfordjournals.qjmed.a067955\",\"intent\":[\"background\"],\"isInfluential\":false,\"paperId\":\"7648a4f73c57c7d7596398bf3555df54df1d647e\",\"title\":\"The clinical spectrum and diagnosis of Mycoplasma pneumoniae infection.\",\"url\":\"https://www.semanticscholar.org/paper/7648a4f73c57c7d7596398bf3555df54df1d647e\",\"venue\":\"The Quarterly journal of medicine\",\"year\":1986},{\"arxivId\":null,\"authors\":[{\"authorId\":\"3256252\",\"name\":\"Marjaleena Koskiniemi\",\"url\":\"https://www.semanticscholar.org/author/3256252\"}],\"doi\":\"10.1093/clinids/17.Supplement_1.S52\",\"intent\":[],\"isInfluential\":false,\"paperId\":\"a95f831a7692e2534f847833f0aa53b20393a86e\",\"title\":\"CNS Manifestations Associated with Mycoplasma pneumoniae Infections: Summary of Cases at the University of Helsinki and Review\",\"url\":\"https://www.semanticscholar.org/paper/a95f831a7692e2534f847833f0aa53b20393a86e\",\"venue\":\"Clinical infectious diseases : an official publication of the Infectious Diseases Society of America\",\"year\":1993},{\"arxivId\":null,\"authors\":[{\"authorId\":\"40663912\",\"name\":\"Sheldon Paul Stone\",\"url\":\"https://www.semanticscholar.org/author/40663912\"}],\"doi\":\"10.1016/S0140-6736(05)61363-3\",\"intent\":[\"background\"],\"isInfluential\":false,\"paperId\":\"724c735b5bd8d71752f23ae904c60fc63d900117\",\"title\":\"Community-acquired pneumonia\",\"url\":\"https://www.semanticscholar.org/paper/724c735b5bd8d71752f23ae904c60fc63d900117\",\"venue\":\"The Lancet\",\"year\":1998},{\"arxivId\":null,\"authors\":[{\"authorId\":\"82754722\",\"name\":\"W A Clyde\",\"url\":\"https://www.semanticscholar.org/author/82754722\"}],\"doi\":null,\"intent\":[\"background\"],\"isInfluential\":true,\"paperId\":\"2eefa64863ac30450246e182c7d3493c86182409\",\"title\":\"Clinical overview of typical Mycoplasma pneumoniae infections.\",\"url\":\"https://www.semanticscholar.org/paper/2eefa64863ac30450246e182c7d3493c86182409\",\"venue\":\"Clinical infectious diseases : an official publication of the Infectious Diseases Society of America\",\"year\":1993},{\"arxivId\":null,\"authors\":[{\"authorId\":\"31736784\",\"name\":\"Mark J. Sands\",\"url\":\"https://www.semanticscholar.org/author/31736784\"},{\"authorId\":\"49862401\",\"name\":\"J. E. Satz\",\"url\":\"https://www.semanticscholar.org/author/49862401\"},{\"authorId\":\"26866123\",\"name\":\"Wallace E. Turner\",\"url\":\"https://www.semanticscholar.org/author/26866123\"},{\"authorId\":\"4878158\",\"name\":\"L A Soloff\",\"url\":\"https://www.semanticscholar.org/author/4878158\"}],\"doi\":\"10.7326/0003-4819-86-5-544\",\"intent\":[],\"isInfluential\":false,\"paperId\":\"af7397fb87a497441aa95be5b38c49d65b00954d\",\"title\":\"Pericarditis and perimyocarditis associated with active Mycoplasma pneumoniae infection.\",\"url\":\"https://www.semanticscholar.org/paper/af7397fb87a497441aa95be5b38c49d65b00954d\",\"venue\":\"Annals of internal medicine\",\"year\":1977},{\"arxivId\":null,\"authors\":[{\"authorId\":\"145312176\",\"name\":\"James P. Luby\",\"url\":\"https://www.semanticscholar.org/author/145312176\"}],\"doi\":null,\"intent\":[\"background\"],\"isInfluential\":false,\"paperId\":\"1d199dac28fcc2f8c29aac1c4f3d3c3b3b56f09a\",\"title\":\"Pneumonia caused by Mycoplasma pneumoniae infection.\",\"url\":\"https://www.semanticscholar.org/paper/1d199dac28fcc2f8c29aac1c4f3d3c3b3b56f09a\",\"venue\":\"Clinics in chest medicine\",\"year\":1991},{\"arxivId\":null,\"authors\":[{\"authorId\":\"13023790\",\"name\":\"A al Rashed\",\"url\":\"https://www.semanticscholar.org/author/13023790\"}],\"doi\":null,\"intent\":[\"background\"],\"isInfluential\":false,\"paperId\":\"65fbd46c00a7e19820e9051627906d4c805ab105\",\"title\":\"Role of Mycoplasma pneumoniae in acute respiratory-tract infections in Saudi paediatric patients.\",\"url\":\"https://www.semanticscholar.org/paper/65fbd46c00a7e19820e9051627906d4c805ab105\",\"venue\":\"Annals of tropical medicine and parasitology\",\"year\":1998}],\"title\":\"Clinical features of culture-proven Mycoplasma pneumoniae infections at King Abdulaziz University Hospital, Jeddah, Saudi Arabia\",\"topics\":[{\"topic\":\"Infection\",\"topicId\":\"1032\",\"url\":\"https://www.semanticscholar.org/topic/1032\"},{\"topic\":\"Bronchiolitis\",\"topicId\":\"39758\",\"url\":\"https://www.semanticscholar.org/topic/39758\"},{\"topic\":\"Respiratory Tract Infections\",\"topicId\":\"4419\",\"url\":\"https://www.semanticscholar.org/topic/4419\"},{\"topic\":\"Cessation of life\",\"topicId\":\"35\",\"url\":\"https://www.semanticscholar.org/topic/35\"},{\"topic\":\"Fever\",\"topicId\":\"1213\",\"url\":\"https://www.semanticscholar.org/topic/1213\"},{\"topic\":\"Pneumocystis jiroveci pneumonia\",\"topicId\":\"73812\",\"url\":\"https://www.semanticscholar.org/topic/73812\"},{\"topic\":\"Wheezing\",\"topicId\":\"14232\",\"url\":\"https://www.semanticscholar.org/topic/14232\"},{\"topic\":\"Malaise\",\"topicId\":\"12701\",\"url\":\"https://www.semanticscholar.org/topic/12701\"},{\"topic\":\"Accident and Emergency department\",\"topicId\":\"9559\",\"url\":\"https://www.semanticscholar.org/topic/9559\"},{\"topic\":\"Specimen\",\"topicId\":\"32605\",\"url\":\"https://www.semanticscholar.org/topic/32605\"},{\"topic\":\"Respiratory tract structure\",\"topicId\":\"17013\",\"url\":\"https://www.semanticscholar.org/topic/17013\"},{\"topic\":\"Scientific Publication\",\"topicId\":\"347\",\"url\":\"https://www.semanticscholar.org/topic/347\"},{\"topic\":\"Upper Respiratory Infections\",\"topicId\":\"57795\",\"url\":\"https://www.semanticscholar.org/topic/57795\"},{\"topic\":\"Chart evaluation by healthcare professional\",\"topicId\":\"297558\",\"url\":\"https://www.semanticscholar.org/topic/297558\"},{\"topic\":\"Coughing\",\"topicId\":\"280\",\"url\":\"https://www.semanticscholar.org/topic/280\"},{\"topic\":\"Patients\",\"topicId\":\"153409\",\"url\":\"https://www.semanticscholar.org/topic/153409\"}],\"url\":\"https://www.semanticscholar.org/paper/dee1269e42a568e8ab3afa5317c45ae4007ebe81\",\"venue\":\"BMC infectious diseases\",\"year\":2001}\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "url = https://api.semanticscholar.org/v1/paper/d1aafb70c066a2068b02786f8929fd9c900897fb\n",
       "result = \n"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "source": "user"
    },
    {
     "data": {
      "text/plain": [
       "\"{\"abstract\":\"ObjectiveThis retrospective chart review describes the epidemiology and clinical features of 40 patients with culture-proven Mycoplasma pneumoniae infections at King Abdulaziz University Hospital, Jeddah, Saudi Arabia.MethodsPatients with positive M. pneumoniae cultures from respiratory specimens from January 1997 through December 1998 were identified through the Microbiology records. Charts of patients were reviewed.Results40 patients were identified, 33 (82.5%) of whom required admission. Most infections (92.5%) were community-acquired. The infection affected all age groups but was most common in infants (32.5%) and pre-school children (22.5%). It occurred yea...\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "source": "user"
    }
   ],
   "source": [
    "val url = \"https://api.semanticscholar.org/v1/paper/d1aafb70c066a2068b02786f8929fd9c900897fb\"\n",
    "val result = scala.io.Source.fromURL(url).mkString\n",
    "println(result)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Apache Toree - Scala",
   "language": "scala",
   "name": "apache_toree_scala"
  },
  "language_info": {
   "codemirror_mode": "text/x-scala",
   "file_extension": ".scala",
   "mimetype": "text/x-scala",
   "name": "scala",
   "pygments_lexer": "scala",
   "version": "2.11.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}