pythontests.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What we have to do for each trialregistry and number match we find in one PRREG field:\n",
    "- create separate prereg nodes for each trialregistry and number match!\n",
    "- we still need to add any urls and dois!\n",
    "- but what if the url/doi are for the same trialnumber? We need to match them up!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('drks', 'DRKS00007824')]\n",
      "[]\n",
      "[('drks', 'DRKS00015308')]\n",
      "[('drks', 'DRKS00013206')]\n",
      "[('drks', 'DRKS00007687')]\n",
      "[('drks', 'DRKS00022819')]\n",
      "[]\n",
      "[]\n",
      "[]\n",
      "[]\n",
      "[]\n",
      "[]\n",
      "[('clinical-trials-gov', 'NCT03418142')]\n",
      "[('drks', 'DRKS00022867')]\n",
      "[('drks', 'DRKS00013309'), ('prospero', 'CRD42018084057')]\n",
      "[('drks', 'DRKS00013206')]\n",
      "[('drks', 'DRKS00007824')]\n",
      "[('drks', 'DRKS00020564')]\n",
      "[('clinical-trials-gov', 'NCT02375308')]\n"
     ]
    }
   ],
   "source": [
    "## find trial numbers in PRREG and match to registry\n",
    "\n",
    "prregs = (\n",
    "    \"|u https://drks.de/search/de/trial/DRKS00007824\",\n",
    "    \"|u https://osf.io/pwjhx |d 10.17605/OSF.IO/PWJHX\",\n",
    "    \"|u https://drks.de/search/de/trial/DRKS00015308\",\n",
    "    \"|u https://drks.de/search/en/trial/DRKS00013206 |d  |i DRKS-ID: DRKS00013206\",\n",
    "    \"|u https://drks.de/search/de/trial/DRKS00007687 |i DRKS-ID DRKS00007687\",\n",
    "    \"|u https://drks.de/search/en/trial/DRKS00022819 |d  |i DRKS00022819\",\n",
    "    \"|u https://aspredicted.org/mw3aq\",\n",
    "    \"|u https://osf.io/2gpn9  |d  |i \",\n",
    "    \"!|u https://www.crd.york.ac.uk/PROSPERO/display_record.php?RecordID=221753  |d  |i \",\n",
    "    \"|u https://osf.io/tz7hy?view only=b42d75e6b88d4996a5cd1637220e42ef |d  |i Study 2\",\n",
    "    \"|u https://osf.io/5r3yp |d https://doi.org/10.17605/OSF.IO/5R3YP |i first preregistration\",\n",
    "    \"|u https://osf.io/98w3h |d https://doi.org/10.17605/OSF.IO/98W3H |i amended preregistration\",\n",
    "    \"|u https://clinicaltrials.gov/ct2/show/NCT03418142\",\n",
    "    \"|u https://www.drks.de/drks web/navigate.do?navigationId=trial.HTML&amp;TRIAL ID=DRKS00022867\", \n",
    "    \"|u https://drks.de/search/de/trial/DRKS00013309 |d  |i PROSPERO (https://www.crd.york.ac.uk/prospero; CRD42018084057; 2018/02/01), German Clinical Trials Register (www.drks.de; DRKS00013309; 2018/01/23).\", \n",
    "    \"|u https://drks.de/search/en/trial/DRKS00013206 |d  |i DRKS-ID: DRKS00013206\",\n",
    "    \"|u https://drks.de/search/de/trial/DRKS00007824\", \n",
    "    \"https://drks.de/search/en/trial/DRKS00020564\",\n",
    "    \"|u https://clinicaltrials.gov/ct2/show/NCT02375308?term=NCT02375308&amp;draw=2&amp;rank=1\"\n",
    "    )\n",
    "\n",
    "import re\n",
    "\n",
    "# a set of trial number regexes and the corresponding registry uri in \n",
    "# https://w3id.org/zpid/vocabs/trialregs/\n",
    "trial_number_regexes = [\n",
    "    (\"DRKS\\d+\", \"drks\"),\n",
    "    (\"CRD\\d+\", \"prospero\"),\n",
    "    (\"ISRCTN\\d+\", \"srctn\"),\n",
    "    (\"NCT\\d+\", \"clinical-trials-gov\"),\n",
    "    (\"actrn\\d+\", \"anzctr\"),\n",
    "    (\"(?i)chictr[-a-z]*\\d+\", \"chictr\"),\n",
    "    (\"kct\\d+\", \"cris\"),\n",
    "    (\"ctri[\\d/]+\", \"clinical-trial-registry-india\"),\n",
    "    (\"\\d{4}-\\d+-\\d+\", \"euctr\"),\n",
    "    (\"irct[0-9a-z]+\", \"irct\"),\n",
    "    (\"isrctn\\d+\", \"isrctn\"),\n",
    "    # (\"\", \"jma\"),\n",
    "    # (\"\", \"jprn\"),\n",
    "    (\"(?i)(nl|ntr)[-0-9]+\", \"dutch-trial-register\"),\n",
    "    (\"rbr\\d+\", \"rebec\"),\n",
    "    (\"rpcec\\d+\", \"rpec\"),\n",
    "    (\"slctr[\\d/]+\", \"slctr\"),\n",
    "    (\"tctr\\d+\", \"tctr\"),\n",
    "    (\"umin\\d+\", \"umin-japan\"),\n",
    "    (\"u[\\d-]+\", \"utn\")\n",
    "]\n",
    "\n",
    "def add_trials_as_preregs(prereg_string, trial_number_regexes):\n",
    "    \"\"\"Checks the PRREG field for trial numbers and adds them as separate preregistrations per number, adding the recognzed registry, too.\n",
    "    TODO: also checks any existing Preregistration nodes to see if a trial is already listed via its url, and adding the trialnumber and registry to that node, otherwise creating a new Preregistration node.\n",
    "    \"\"\"\n",
    "    # a string may contain several trial numbers from different registries. \n",
    "    # match all of them!\n",
    "    trailnumber_matches = []\n",
    "    for trial_number_regex, trialreg in trial_number_regexes:\n",
    "        # match = trial_number_regex.search(prereg_string)\n",
    "        # change to use a string for the regex, adding re.compile() here only once:\n",
    "        match = re.compile(trial_number_regex).search(prereg_string)\n",
    "        if match:\n",
    "            trailnumber_matches.append((trialreg, match.group()))\n",
    "            # print(match.group() + \" matches registry: \" + trialreg)\n",
    "    print(trailnumber_matches)\n",
    "    return trailnumber_matches\n",
    "\n",
    "\n",
    "    # for trial_number_regex, trialreg in trial_number_regexes:\n",
    "    #     match = trial_number_regex.search(prereg_string)\n",
    "    #     if match:\n",
    "    #         print(match.group() + \" matches registry: \" + trialreg)\n",
    "    #         #return trialreg, match.group()\n",
    "    \n",
    "\n",
    "for prreg in prregs:\n",
    "    # print(\"this string: \" + prreg + \" matches: \")\n",
    "    add_trials_as_preregs(prreg, trial_number_regexes)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 377,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==Exact Matches==\n",
      "\n",
      "These records are definitely identical:\n",
      "0368936, 0368935, \n",
      "\n",
      "These records are definitely identical:\n",
      "000002, 000003, \n",
      "\n",
      "==Possible Matches==\n",
      "For these, only the list of authors is different, but the title and first author are the same:\n",
      "\n",
      "check these records, they may be identical:\n",
      "0360687, 0368936, 0368935, \n",
      "\n",
      "check these records, they may be identical:\n",
      "000001, 000002, 000003, \n"
     ]
    }
   ],
   "source": [
    "# generate a title key for deduplication (Dublettencheck und Werksextraktion)\n",
    "import json\n",
    "import re\n",
    "\n",
    "records_before = [\n",
    "    # this first record is NOT a definite duplicate of the second one, because the last name of the second author is different: it should appear in the list of possible duplicates:\n",
    "    {\"DFK\":\"0360687\", \"mainTitle\": \"Interkulturelle Kompetenz der Möglichkeiten: Kritische Betrachtung eines Konstrukts und mehr - mit Aenderungen am Maß\", \"subtitle\": \"\", \"PY\":\"2018\", \"authors\": [{\"familyname\":\"Genkova\", \"givenname\":\"Petia\"},{\"familyname\":\"Maler\", \"givenname\":\"Pedro\"}]},\n",
    "    # the following two are exact duplicates of each other - they should end up in a \"definite duplicates\" list:\n",
    "    {\"DFK\":\"0368936\", \"mainTitle\": \"Interkulturelle Kompetenz der Möglichkeiten\", \"subtitle\": \"Kritische Betrachtung eines Konstrukts and mehr, mit Änderungen am Mass\", \"PY\":\"2020\", \"authors\": [{\"familyname\":\"Genkova\", \"givenname\":\"Petia\"},{\"familyname\":\"Müller\", \"givenname\":\"H.\"}]},\n",
    "    {\"DFK\":\"0368935\", \"mainTitle\": \"Interkulturelle Kompetenz der Möglichkeiten\", \"subtitle\": \"Kritische Betrachtung eines Konstrukts and mehr, mit Änderungen am Mass\", \"PY\":\"2020\", \"authors\": [{\"familyname\":\"Genkova\", \"givenname\":\"P. A.\"},{\"familyname\":\"Mueller\", \"givenname\":\"H.\"}]},\n",
    "    # the following three are also near duplicates of each other - the first should end up in a \"possible duplicates\" list, the other two in a \"definite duplicates\" list:\n",
    "    {\"DFK\":\"000001\",\"mainTitle\": \"Homogenität.\", \"subtitle\": \"Ein Maß für Ärger\", \"PY\":\"2022\", \"authors\": [{\"familyname\":\"Genkova\", \"givenname\":\"P.\"},{\"familyname\":\"Mueller\", \"givenname\":\"Pedro\"}]},\n",
    "    # these two are exact duplicates - they have the same title key, first author key and all authors key:\n",
    "    {\"DFK\":\"000002\",\"mainTitle\": \"Homogenitaet: Ein Maß für Ärger\", \"subtitle\": \"\", \"PY\":\"2022\", \"authors\": [{\"familyname\":\"Genkova\", \"givenname\":\"Petia\"},{\"familyname\":\"Mueller\", \"givenname\":\"Heinz\"}]},\n",
    "    {\"DFK\":\"000003\",\"mainTitle\": \"Homogenitaet: Ein Mass für AErger\", \"subtitle\": \"\", \"PY\":\"2022\", \"authors\": [{\"familyname\":\"Genkova\", \"givenname\":\"P. A.\"},{\"familyname\":\"Müller\", \"givenname\":\"H.\"}]}\n",
    "]\n",
    "\n",
    "umlaut_map = {\"ö\":\"oe\", \"ä\":\"ae\", \"ü\":\"ue\", \"ß\":\"ss\"}\n",
    "special_char_removal = re.compile(\"[^a-z&0-0]\")\n",
    "and_replace_map = {\"and\": \"&\", \"und\": \"&\"}\n",
    "\n",
    "def generate_title_key(title):\n",
    "    key = title.casefold().translate(str.maketrans(umlaut_map))\n",
    "    # replace the ands_and_unds:\n",
    "    for word, initial in and_replace_map.items():\n",
    "        key = re.sub(r\"\\b{}\\b\".format(word), initial, key)\n",
    "    # remove special characters:\n",
    "    key = special_char_removal.sub(\"\", key)\n",
    "    return key\n",
    "\n",
    "def generate_single_author_key(givenname, familyname):\n",
    "    # we want only the first letter of the given name:\n",
    "    givenname = givenname[0]\n",
    "    # concatenate the two:\n",
    "    key = familyname + givenname\n",
    "    # make it lowercase and replace umlauts:\n",
    "    key = key.casefold().translate(str.maketrans(umlaut_map))\n",
    "    # todo: sometimes journals mangle author names with umlauts by simply removing the dots, such that Müller becomes Muller, not Mueller. What can we do about that?\n",
    "    return key\n",
    "\n",
    "def generate_all_authors_key(authors):\n",
    "    # make an empty string:\n",
    "    key = \"\"\n",
    "    # go through all authors and add their keys to the string:\n",
    "    for author in authors:\n",
    "        # get the keys:\n",
    "        givenname = author[\"givenname\"]\n",
    "        familyname = author[\"familyname\"]\n",
    "        # generate the key for this author:\n",
    "        author_key = generate_single_author_key(givenname, familyname)\n",
    "        # add it to the string:\n",
    "        key += author_key\n",
    "    return key\n",
    "\n",
    "\n",
    "def generate_keys_for_all(records):\n",
    "    # go through all records and generate a key for each:\n",
    "    for record in records:\n",
    "        # generate the key for the current record and add it to the list (dfk_list) of all records with their keys:\n",
    "        title_key = generate_title_key(record[\"mainTitle\"] + \" \" + record[\"subtitle\"])\n",
    "        # add the new key to the existing record:\n",
    "        record[\"title_key\"] = title_key\n",
    "        first_author_key = generate_single_author_key(record[\"authors\"][0][\"givenname\"], record[\"authors\"][0][\"familyname\"])\n",
    "        record[\"first_author_key\"] = first_author_key\n",
    "        all_authors_key = generate_all_authors_key(record[\"authors\"])\n",
    "        # add to the record:\n",
    "        record[\"all_authors_key\"] = all_authors_key\n",
    "        #print(record)\n",
    "        # save the record in a file named records_with_keys.json\n",
    "        with open(\"records_with_keys.json\", \"w\", encoding=\"utf-8\") as f:\n",
    "           json.dump(records, f, indent=2)\n",
    "\n",
    "    return records\n",
    "####\n",
    "####\n",
    "\n",
    "records = [\n",
    "  {\n",
    "    \"DFK\": \"0360687\",\n",
    "    \"mainTitle\": \"Interkulturelle Kompetenz der M\\u00f6glichkeiten: Kritische Betrachtung eines Konstrukts und mehr - mit Aenderungen am Ma\\u00df\",\n",
    "    \"subtitle\": \"\",\n",
    "    \"PY\": \"2018\",\n",
    "    \"authors\": [\n",
    "      {\n",
    "        \"familyname\": \"Genkova\",\n",
    "        \"givenname\": \"Petia\"\n",
    "      },\n",
    "      {\n",
    "        \"familyname\": \"Maler\",\n",
    "        \"givenname\": \"Pedro\"\n",
    "      }\n",
    "    ],\n",
    "    \"title_key\": \"interkulturellekompetenzdermoeglichkeitenkritischebetrachtungeineskonstrukts&mehrmitaenderungenammass\",\n",
    "    \"first_author_key\": \"genkovap\",\n",
    "    \"all_authors_key\": \"genkovapmalerp\"\n",
    "  },\n",
    "  {\n",
    "    \"DFK\": \"0368936\",\n",
    "    \"mainTitle\": \"Interkulturelle Kompetenz der M\\u00f6glichkeiten\",\n",
    "    \"subtitle\": \"Kritische Betrachtung eines Konstrukts and mehr, mit \\u00c4nderungen am Mass\",\n",
    "    \"PY\": \"2020\",\n",
    "    \"authors\": [\n",
    "      {\n",
    "        \"familyname\": \"Genkova\",\n",
    "        \"givenname\": \"Petia\"\n",
    "      },\n",
    "      {\n",
    "        \"familyname\": \"M\\u00fcller\",\n",
    "        \"givenname\": \"H.\"\n",
    "      }\n",
    "    ],\n",
    "    \"title_key\": \"interkulturellekompetenzdermoeglichkeitenkritischebetrachtungeineskonstrukts&mehrmitaenderungenammass\",\n",
    "    \"first_author_key\": \"genkovap\",\n",
    "    \"all_authors_key\": \"genkovapmuellerh\"\n",
    "  },\n",
    "  {\n",
    "    \"DFK\": \"0368935\",\n",
    "    \"mainTitle\": \"Interkulturelle Kompetenz der M\\u00f6glichkeiten\",\n",
    "    \"subtitle\": \"Kritische Betrachtung eines Konstrukts and mehr, mit \\u00c4nderungen am Mass\",\n",
    "    \"PY\": \"2020\",\n",
    "    \"authors\": [\n",
    "      {\n",
    "        \"familyname\": \"Genkova\",\n",
    "        \"givenname\": \"P. A.\"\n",
    "      },\n",
    "      {\n",
    "        \"familyname\": \"Mueller\",\n",
    "        \"givenname\": \"H.\"\n",
    "      }\n",
    "    ],\n",
    "    \"title_key\": \"interkulturellekompetenzdermoeglichkeitenkritischebetrachtungeineskonstrukts&mehrmitaenderungenammass\",\n",
    "    \"first_author_key\": \"genkovap\",\n",
    "    \"all_authors_key\": \"genkovapmuellerh\"\n",
    "  },\n",
    "  {\n",
    "    \"DFK\": \"000001\",\n",
    "    \"mainTitle\": \"Homogenit\\u00e4t.\",\n",
    "    \"subtitle\": \"Ein Ma\\u00df f\\u00fcr \\u00c4rger\",\n",
    "    \"PY\": \"2022\",\n",
    "    \"authors\": [\n",
    "      {\n",
    "        \"familyname\": \"Genkova\",\n",
    "        \"givenname\": \"P.\"\n",
    "      },\n",
    "      {\n",
    "        \"familyname\": \"Mueller\",\n",
    "        \"givenname\": \"Pedro\"\n",
    "      }\n",
    "    ],\n",
    "    \"title_key\": \"homogenitaeteinmassfueraerger\",\n",
    "    \"first_author_key\": \"genkovap\",\n",
    "    \"all_authors_key\": \"genkovapmuellerp\"\n",
    "  },\n",
    "  {\n",
    "    \"DFK\": \"000002\",\n",
    "    \"mainTitle\": \"Homogenitaet: Ein Ma\\u00df f\\u00fcr \\u00c4rger\",\n",
    "    \"subtitle\": \"\",\n",
    "    \"PY\": \"2022\",\n",
    "    \"authors\": [\n",
    "      {\n",
    "        \"familyname\": \"Genkova\",\n",
    "        \"givenname\": \"Petia\"\n",
    "      },\n",
    "      {\n",
    "        \"familyname\": \"Mueller\",\n",
    "        \"givenname\": \"Heinz\"\n",
    "      }\n",
    "    ],\n",
    "    \"title_key\": \"homogenitaeteinmassfueraerger\",\n",
    "    \"first_author_key\": \"genkovap\",\n",
    "    \"all_authors_key\": \"genkovapmuellerh\"\n",
    "  },\n",
    "  {\n",
    "    \"DFK\": \"000003\",\n",
    "    \"mainTitle\": \"Homogenitaet: Ein Mass f\\u00fcr AErger\",\n",
    "    \"subtitle\": \"\",\n",
    "    \"PY\": \"2022\",\n",
    "    \"authors\": [\n",
    "      {\n",
    "        \"familyname\": \"Genkova\",\n",
    "        \"givenname\": \"P. A.\"\n",
    "      },\n",
    "      {\n",
    "        \"familyname\": \"M\\u00fcller\",\n",
    "        \"givenname\": \"H.\"\n",
    "      }\n",
    "    ],\n",
    "    \"title_key\": \"homogenitaeteinmassfueraerger\",\n",
    "    \"first_author_key\": \"genkovap\",\n",
    "    \"all_authors_key\": \"genkovapmuellerh\"\n",
    "  }\n",
    "]\n",
    "\n",
    "def find_duplicate_dfks(records):\n",
    "    # Dictionary to store perfect matches based on title_key, first_author_key, and all_authors_key\n",
    "    # it will be structured as follows:\n",
    "    # { \n",
    "    #   (title_key, first_author_key, all_authors_key): [dfk1, dfk2, ...],\n",
    "    #   ...\n",
    "    # }\n",
    "    # where dfk1, dfk2, ... are the DFKs of the records that match the key\n",
    "    # and (title_key, first_author_key, all_authors_key) is a list of the keys that are the same for the records (dfks) listed as values\n",
    "    perfect_matches = {}\n",
    "\n",
    "    for record in records:\n",
    "        dfk = record[\"DFK\"]\n",
    "        title_key = record[\"title_key\"]\n",
    "        first_author_key = record[\"first_author_key\"]\n",
    "        all_authors_key = record[\"all_authors_key\"]\n",
    "\n",
    "        if (title_key, first_author_key, all_authors_key) in perfect_matches:\n",
    "            perfect_matches[(title_key, first_author_key, all_authors_key)].append(dfk)\n",
    "        else:\n",
    "            perfect_matches[(title_key, first_author_key, all_authors_key)] = [dfk]\n",
    "\n",
    "    # Dictionary to store less perfect matches based on title_key and first_author_key\n",
    "    possible_matches = {}\n",
    "\n",
    "    for (title_key, first_author_key, all_authors_key), dfk_list in perfect_matches.items():\n",
    "        for dfk in dfk_list:\n",
    "            if (title_key, first_author_key) in possible_matches:\n",
    "                possible_matches[(title_key, first_author_key)].append(dfk)\n",
    "            else:\n",
    "                possible_matches[(title_key, first_author_key)] = [dfk]\n",
    "\n",
    "    perfect_matches = {k: v for k, v in perfect_matches.items() if len(v) > 1}\n",
    "    possible_matches = {k: v for k, v in possible_matches.items() if len(v) > 1}\n",
    "\n",
    "    return perfect_matches, possible_matches\n",
    "\n",
    "\n",
    "perfect_matches, possible_matches = find_duplicate_dfks(records)\n",
    "\n",
    "# print(perfect_matches)\n",
    "# print(possible_matches)\n",
    "\n",
    "print(\"==Exact Matches==\")\n",
    "# print in a structured way by going through the dictionary:\n",
    "for item in perfect_matches.items():\n",
    "    print(\"\\nThese records are definitely identical:\")\n",
    "    for dfk in item[1]:\n",
    "        # print all in the same line, comma separated:\n",
    "        print(dfk, end=\", \")\n",
    "    print()\n",
    "\n",
    "print(\"\\n==Possible Matches==\")\n",
    "print(\"For these, only the list of authors is different, but the title and first author are the same:\")\n",
    "for item in possible_matches.items():\n",
    "    print(\"\\ncheck these records, they may be identical:\")\n",
    "    for dfk in item[1]:\n",
    "        print(dfk, end=\", \")\n",
    "    print()\n",
    "\n",
    "\n",
    "# print(\"Perfect Matches:\")\n",
    "# for (title_key, first_author_key, all_authors_key), dfk_list in perfect_matches.items():\n",
    "#     print(f\"Title Key: {title_key}, First Author Key: {first_author_key}, All Authors Key: {all_authors_key}\")\n",
    "#     for dfk in dfk_list:\n",
    "#         print(f\"- DFK: {dfk}\")\n",
    "#     print()\n",
    "\n",
    "# print(\"\\nLess Perfect Matches:\")\n",
    "# for (title_key, first_author_key), dfk_list in less_perfect_matches.items():\n",
    "#     print(f\"Title Key: {title_key}, First Author Key: {first_author_key}\")\n",
    "#     for dfk in dfk_list:\n",
    "#         print(f\"- DFK: {dfk}\")\n",
    "#     print()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Model:\n",
    "\n",
    "```r\n",
    "<Work> a bf:Work ;\n",
    "    bf:hasInstance [\n",
    "\n",
    "        # the partOf relationship is between article instance and journal instance:\n",
    "        a bf:Instance ;\n",
    "        bf:issuance issuanceType:JournalArticle ;\n",
    "        # Relationship bnode with all the info about the journal:\n",
    "        bflc:relationship [\n",
    "            a bflc:Relationship ;\n",
    "            bflc:relation relations:isArticleInJournal ;\n",
    "            # biblio info: issue, vol, pages, articleno:\n",
    "            pxp:inVolume \"91\"; # taken from field JBD\n",
    "            pxp:inIssue \"1\"; # taken from field JHFT\n",
    "            pxp:pageStart \"1\"; # taken from combined record (split field PAGE)\n",
    "            pxp:pageEnd \"26\";\n",
    "            # or pxp:articleNumber \"No. 1234455\"; # taken from field PAGE\n",
    "            bf:hasSeries [\n",
    "                a bf:Instance ;\n",
    "                bf:title [a bf:Title ; rdfs:label \"Journal of the American Chemical Society\" ] ;\n",
    "                bf:instanceOf [a bf:Work, bf:Serial;\n",
    "                    bf:title [a bf:Title ; rdfs:label \"Journal of the American Chemical Society\" ] ;\n",
    "                    bf:issuance issuanceType:Periodical ;\n",
    "                    bf:identifiedBy [\n",
    "                        a bf:Issn ;\n",
    "                        rdf:value \"0002-7863\" ;\n",
    "                    ]\n",
    "                ]\n",
    "            ]\n",
    "        ]\n",
    "    ]\n",
    "```\n",
    "\n",
    "Note: In PSYNDEXER, we link an article's \"Instance bundle\" - actually several instances with different \"media types\" connected by bf:otherPhysicalFormat - to the journal \"hub\". \n",
    "But how can we do this in the model?\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# linking journal and location in it:\n",
    "records = [\n",
    "    {'JT': \"Translational Neuroscience\", 'ISSN': \"2081-3856\", 'EISSN': \"2081-6936\",\n",
    "     'JBD':\"8\", 'JHFT':\"1\", 'PAGE':\"182-190\", 'MT': 'Print', 'MT2': 'Online Medium'},\n",
    "     # Articleno in PAGE:\n",
    "    {'JT': \"Philosophical Transactions of the Royal Society - Series B\", 'ISSN': \"0962-8436\", 'EISSN': \"1471-2970\",\n",
    "        'JBD':\"373\", 'PAGE':\"No. 20170151\"},\n",
    "    # no JHFT, Articleno in PAGE:\n",
    "    {'JT': \"Frontiers in Psychiatry\", 'ISSN': \"1664-0640\",\n",
    "        'JBD':\"9\", 'PAGE':\"No. 114\"},\n",
    "    # no EISSN:\n",
    "    {'JT': \"Wissenschaftliche Zeitschrift der Humboldt-Universität zu Berlin - Gesellschaftswissenschaftliche Reihe\", 'ISSN': \"0522-9855\",\n",
    "        'JBD':\"36\", 'JHFT':\"10\", 'PAGE':\"952-955\", 'MT': 'Print'},\n",
    "]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Language guessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "de\n"
     ]
    }
   ],
   "source": [
    "import langid\n",
    "langid.set_languages([\"de\", \"en\"])\n",
    "\n",
    "def guess_language(string_in_language):\n",
    "    return (langid.classify(string_in_language)[0])\n",
    "\n",
    "# print(langid.classify(\"Zur transgenerationalen Traumatisierung\"))\n",
    "# print(langid.classify(\"Ätiologie und Ansätze für die Therapie\"))\n",
    "\n",
    "# print(langid.classify(\"Zur transgenerationalen Traumatisierung\")[0])\n",
    "print(guess_language(\"\\\"A 'true' artist may draw mountainous seas!\\\" - Eine Würdigung von Paul Watzlawick zu seinem 100. Geburtstag\"))\n",
    "# print(guess_language(\"What does it mean for children's development whether and to what extent both parents are employed and they accordingly spend part of the day outside the family? And how should care for young children be structured? Research findings from Developmental Psychology provide some answers. (translated by DeepL)\"))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Checking strings for non-letters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "yes\n"
     ]
    }
   ],
   "source": [
    "\n",
    "language = \"EnglishX$X$\"\n",
    "# check if string contains any non-letter character:\n",
    "if not language.isalpha():\n",
    "    print(\"yes\")\n",
    "else:\n",
    "    print(\"no\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reconciling affiliation with ror"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Zagreb Childrenğt;'s Hospital, Pediatric Clinic\n",
      "Test Zagreb Childrenğt;'s Hospital, Pediatric Clinic more\n",
      "p &lt; .05\n",
      " &lt; \n",
      "haha ∞ ™\n",
      "Geoffrey Jefferson\n"
     ]
    }
   ],
   "source": [
    "import requests_cache\n",
    "from datetime import timedelta\n",
    "# from mappings import geonames_countries\n",
    "# from mappings import abstract_origins\n",
    "# from mappings import dd_codes\n",
    "\n",
    "# for reconciling affiliation strings with ror api:\n",
    "ROR_API_URL = \"https://api.ror.org/organizations?affiliation=\"\n",
    "\n",
    "# for getting data about a known id:\n",
    "ROR_API_LOOKUP_URL = \"https://api.ror.org/organizations/\"\n",
    "\n",
    "from modules.mappings import dd_codes\n",
    "\n",
    "def replace_encodings(text):\n",
    "    for case in dd_codes:\n",
    "        text = text.replace(case[0], case[1]) \n",
    "    return text\n",
    "\n",
    "urls_expire_after = {\n",
    "    # Custom cache duration per url, 0 means \"don't cache\"\n",
    "    # f'{SKOSMOS_URL}/rest/v1/label?uri=https%3A//w3id.org/zpid/vocabs/terms/09183&lang=de': 0,\n",
    "    # f'{SKOSMOS_URL}/rest/v1/label?uri=https%3A//w3id.org/zpid/vocabs/terms/': 0,\n",
    "}\n",
    "\n",
    "session = requests_cache.CachedSession(\n",
    "    \".cache/requests\",\n",
    "    allowable_codes=[200, 404],\n",
    "    expire_after=timedelta(days=30),\n",
    "    urls_expire_after=urls_expire_after,\n",
    ")\n",
    "\n",
    "def get_ror_id_from_api(affiliation_string):\n",
    "    # this function takes a string with an affiliation name and returns the ror id for that affiliation from the ror api\n",
    "    # clean the string to make sure things like \"^DDS\" are replaced:\n",
    "    affiliation_string = replace_encodings(affiliation_string)\n",
    "    #replace_encodings(affiliation_string)\n",
    "    ror_api_url = ROR_API_URL + affiliation_string\n",
    "    # make a request to the ror api:\n",
    "    # ror_api_request = requests.get(ror_api_url)\n",
    "    # make request to api with caching:\n",
    "    ror_api_request = session.get(\n",
    "            ror_api_url, timeout=20\n",
    "    )\n",
    "    # if the request was successful, get the json response:\n",
    "    if ror_api_request.status_code == 200:\n",
    "        ror_api_response = ror_api_request.json()\n",
    "        # check if the response has any hits:\n",
    "        if len(ror_api_response[\"items\"]) > 0:\n",
    "            # if so, get the item with a key value pair of \"chosen\" and \"true\" and return its id:\n",
    "            for item in ror_api_response[\"items\"]:\n",
    "                if item[\"chosen\"] == True:\n",
    "                    return item[\"organization\"][\"id\"]\n",
    "        else:\n",
    "            return None\n",
    "    else:\n",
    "        return None\n",
    "    \n",
    "\n",
    "    # here is a list of affiliation strings to go through:\n",
    "affiliation_strings = [\n",
    "    \"Klinik für Frauenheilkunde und Geburtshilfe, Universitätsklinikum Ulm\",\n",
    "    \"Klinik für Psychososmatische Medizin und Psychotherapie, Universitätsklinikum Ulm\",\n",
    "    \"Sektion Medizinische Psychologie, Universitätsklinikum Ulm\",\n",
    "    \"Klinik für Psychososmatische Medizin und Psychotherapie, Universitätsklinikum Ulm\",\n",
    "    \"Psychology School, Hochschule Fresenius ^DDS University of Applied Sciences, Düsseldorf\",\n",
    "    \"Fakultät Medizin, MSH Medical School Hamburg ^DDS University of Applied Sciences and Medical University, Hamburg\",\n",
    "    \"Fakultät Medizin, MSH Medical School Hamburg ^DDS University of Applied Sciences and Medical University, Hamburg\",\n",
    "    \"Department of Child and Adolescent Psychiatry, Psychosomatics and Psychotherapy; LVR Klinikum Essen; University Hospital Essen; University of Duisburg-Essen; Essen\",\n",
    "    \"Child and Adolescent Psychiatry/Psychology, Erasmus Medical Center Rotterdam\"\n",
    "]\n",
    "\n",
    "# use the function to get the ror id for each affiliation string:\n",
    "\n",
    "# for affiliation_string in affiliation_strings:\n",
    "#     print(replace_encodings(affiliation_string) + \": \" + str(get_ror_id_from_api(affiliation_string)))\n",
    "\n",
    "# print(replace_encodings(\"Stimulus ^DDS non psychological interference ^DDL analogy\"))\n",
    "print(replace_encodings(\"Zagreb Children^D&gt;'s Hospital, Pediatric Clinic\"))\n",
    "print(\"Test \" + replace_encodings(\"Zagreb Children^D&gt;'s Hospital, Pediatric Clinic\") + \" more\")\n",
    "print(replace_encodings(\"p &lt; .05\"))\n",
    "print(replace_encodings(' &lt; '))\n",
    "print(replace_encodings('haha ^DIF ^DTM'))\n",
    "print(replace_encodings('Geo^Dffrey Je^Dfferson'))\n",
    "# print(get_ror_id_from_api(\"Klinik für Frauenheilkunde und Geburtshilfe, Universitätsklinikum Ulm\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11\n"
     ]
    }
   ],
   "source": [
    "def utf8len(s):\n",
    "    return len(s.encode('utf-8'))\n",
    "\n",
    "print(utf8len(\"Test string\"))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deprecated: Reconcile with Wikidata\n",
    "\n",
    "The \"reconciler\" package can use other reconciliation APIs, too. Wikidata is the default. \n",
    "To change the API endpoint, call the reconcile() function with the parameter `reconciliation_endpoint=\"https...\"`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/2 [00:00<?, ?it/s]\n"
     ]
    },
    {
     "ename": "JSONDecodeError",
     "evalue": "Expecting value: line 1 column 1 (char 0)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mJSONDecodeError\u001b[0m                           Traceback (most recent call last)",
      "File \u001b[0;32m~/Developement/py-star2bf/.direnv/python-3.10.10/lib/python3.10/site-packages/requests/models.py:971\u001b[0m, in \u001b[0;36mResponse.json\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m    970\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 971\u001b[0m     \u001b[39mreturn\u001b[39;00m complexjson\u001b[39m.\u001b[39;49mloads(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mtext, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m    972\u001b[0m \u001b[39mexcept\u001b[39;00m JSONDecodeError \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m    973\u001b[0m     \u001b[39m# Catch JSON-related errors and raise as requests.JSONDecodeError\u001b[39;00m\n\u001b[1;32m    974\u001b[0m     \u001b[39m# This aliases json.JSONDecodeError and simplejson.JSONDecodeError\u001b[39;00m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.10/lib/python3.10/json/__init__.py:346\u001b[0m, in \u001b[0;36mloads\u001b[0;34m(s, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001b[0m\n\u001b[1;32m    343\u001b[0m \u001b[39mif\u001b[39;00m (\u001b[39mcls\u001b[39m \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m object_hook \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m\n\u001b[1;32m    344\u001b[0m         parse_int \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m parse_float \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m\n\u001b[1;32m    345\u001b[0m         parse_constant \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m object_pairs_hook \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m \u001b[39mnot\u001b[39;00m kw):\n\u001b[0;32m--> 346\u001b[0m     \u001b[39mreturn\u001b[39;00m _default_decoder\u001b[39m.\u001b[39;49mdecode(s)\n\u001b[1;32m    347\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mcls\u001b[39m \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.10/lib/python3.10/json/decoder.py:337\u001b[0m, in \u001b[0;36mJSONDecoder.decode\u001b[0;34m(self, s, _w)\u001b[0m\n\u001b[1;32m    333\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Return the Python representation of ``s`` (a ``str`` instance\u001b[39;00m\n\u001b[1;32m    334\u001b[0m \u001b[39mcontaining a JSON document).\u001b[39;00m\n\u001b[1;32m    335\u001b[0m \n\u001b[1;32m    336\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[0;32m--> 337\u001b[0m obj, end \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mraw_decode(s, idx\u001b[39m=\u001b[39;49m_w(s, \u001b[39m0\u001b[39;49m)\u001b[39m.\u001b[39;49mend())\n\u001b[1;32m    338\u001b[0m end \u001b[39m=\u001b[39m _w(s, end)\u001b[39m.\u001b[39mend()\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.10/lib/python3.10/json/decoder.py:355\u001b[0m, in \u001b[0;36mJSONDecoder.raw_decode\u001b[0;34m(self, s, idx)\u001b[0m\n\u001b[1;32m    354\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mStopIteration\u001b[39;00m \u001b[39mas\u001b[39;00m err:\n\u001b[0;32m--> 355\u001b[0m     \u001b[39mraise\u001b[39;00m JSONDecodeError(\u001b[39m\"\u001b[39m\u001b[39mExpecting value\u001b[39m\u001b[39m\"\u001b[39m, s, err\u001b[39m.\u001b[39mvalue) \u001b[39mfrom\u001b[39;00m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m    356\u001b[0m \u001b[39mreturn\u001b[39;00m obj, end\n",
      "\u001b[0;31mJSONDecodeError\u001b[0m: Expecting value: line 1 column 1 (char 0)",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mJSONDecodeError\u001b[0m                           Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[21], line 46\u001b[0m\n\u001b[1;32m     13\u001b[0m funder_names \u001b[39m=\u001b[39m pd\u001b[39m.\u001b[39mDataFrame(\n\u001b[1;32m     14\u001b[0m     {\n\u001b[1;32m     15\u001b[0m         \u001b[39m\"\u001b[39m\u001b[39mfunder_name\u001b[39m\u001b[39m\"\u001b[39m: [ \u001b[39m\"\u001b[39m\u001b[39mBundesministerium für Bildung und Forschung (BMBF)\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     42\u001b[0m     }\n\u001b[1;32m     43\u001b[0m )\n\u001b[1;32m     45\u001b[0m \u001b[39m# Reconcile against type city (Q515), getting the best match for each item.\u001b[39;00m\n\u001b[0;32m---> 46\u001b[0m reconciled \u001b[39m=\u001b[39m reconcile(funder_names[\u001b[39m\"\u001b[39;49m\u001b[39mfunder_name\u001b[39;49m\u001b[39m\"\u001b[39;49m], reconciliation_endpoint\u001b[39m=\u001b[39;49m\u001b[39m\"\u001b[39;49m\u001b[39mhttp://recon.labs.crossref.org/reconcile\u001b[39;49m\u001b[39m\"\u001b[39;49m)\n\u001b[1;32m     47\u001b[0m \u001b[39m# reconciled = reconcile(funder_names[\"funder_name\"], type_id=\"TerritorialCorporateBodyOrAdministrativeUnit\", property_mapping={\"geographicAreaCode\": test_df[\"Land\"]}, reconciliation_endpoint=\"https://lobid.org/gnd/reconcile/\")\u001b[39;00m\n\u001b[1;32m     48\u001b[0m \u001b[39m# reconciled = reconcile(test_df[\"City\"], type_id=\"Q515\")\u001b[39;00m\n\u001b[1;32m     49\u001b[0m \n\u001b[1;32m     50\u001b[0m \u001b[39m# save the results to a csv file:\u001b[39;00m\n\u001b[1;32m     51\u001b[0m test_df\u001b[39m.\u001b[39mto_csv(\u001b[39m\"\u001b[39m\u001b[39mtest.csv\u001b[39m\u001b[39m\"\u001b[39m)\n",
      "File \u001b[0;32m~/Developement/py-star2bf/.direnv/python-3.10.10/lib/python3.10/site-packages/reconciler/main.py:52\u001b[0m, in \u001b[0;36mreconcile\u001b[0;34m(column_to_reconcile, type_id, top_res, property_mapping, reconciliation_endpoint)\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mreconcile\u001b[39m(\n\u001b[1;32m      7\u001b[0m     column_to_reconcile,\n\u001b[1;32m      8\u001b[0m     type_id\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     11\u001b[0m     reconciliation_endpoint\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mhttps://wikidata.reconci.link/en/api\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[1;32m     12\u001b[0m ):\n\u001b[1;32m     13\u001b[0m \u001b[39m    \u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m     14\u001b[0m \u001b[39m    Reconcile a DataFrame column\u001b[39;00m\n\u001b[1;32m     15\u001b[0m \n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     49\u001b[0m \u001b[39m        ValueError: top_res argument must be one of either 'all' or an integer.\u001b[39;00m\n\u001b[1;32m     50\u001b[0m \u001b[39m    \"\"\"\u001b[39;00m\n\u001b[0;32m---> 52\u001b[0m     input_keys, response \u001b[39m=\u001b[39m return_reconciled_raw(\n\u001b[1;32m     53\u001b[0m         column_to_reconcile,\n\u001b[1;32m     54\u001b[0m         type_id,\n\u001b[1;32m     55\u001b[0m         property_mapping,\n\u001b[1;32m     56\u001b[0m         reconciliation_endpoint,\n\u001b[1;32m     57\u001b[0m     )\n\u001b[1;32m     59\u001b[0m     full_df \u001b[39m=\u001b[39m parse_raw_results(input_keys, response)\n\u001b[1;32m     61\u001b[0m     \u001b[39mif\u001b[39;00m top_res \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mall\u001b[39m\u001b[39m\"\u001b[39m:\n",
      "File \u001b[0;32m~/Developement/py-star2bf/.direnv/python-3.10.10/lib/python3.10/site-packages/reconciler/webutils.py:94\u001b[0m, in \u001b[0;36mreturn_reconciled_raw\u001b[0;34m(df_column, type_id, property_mapping, reconciliation_endpoint)\u001b[0m\n\u001b[1;32m     92\u001b[0m     logger\u001b[39m.\u001b[39mdebug(\u001b[39m'\u001b[39m\u001b[39mreconciling: \u001b[39m\u001b[39m%s\u001b[39;00m\u001b[39m'\u001b[39m, chunk)\n\u001b[1;32m     93\u001b[0m     reconcilable_data \u001b[39m=\u001b[39m json\u001b[39m.\u001b[39mdumps({\u001b[39m\"\u001b[39m\u001b[39mqueries\u001b[39m\u001b[39m\"\u001b[39m: json\u001b[39m.\u001b[39mdumps(chunk)})\n\u001b[0;32m---> 94\u001b[0m     query_result \u001b[39m=\u001b[39m perform_query(reconcilable_data, reconciliation_endpoint)\n\u001b[1;32m     95\u001b[0m     query_results\u001b[39m.\u001b[39mappend(query_result)\n\u001b[1;32m     97\u001b[0m merged_results \u001b[39m=\u001b[39m \u001b[39mdict\u001b[39m(ChainMap(\u001b[39m*\u001b[39mquery_results))\n",
      "File \u001b[0;32m~/Developement/py-star2bf/.direnv/python-3.10.10/lib/python3.10/site-packages/reconciler/webutils.py:47\u001b[0m, in \u001b[0;36mperform_query\u001b[0;34m(query_string, reconciliation_endpoint, max_tries)\u001b[0m\n\u001b[1;32m     45\u001b[0m     tries \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m \u001b[39m1\u001b[39m\n\u001b[1;32m     46\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m---> 47\u001b[0m     query_result \u001b[39m=\u001b[39m response\u001b[39m.\u001b[39;49mjson()\n\u001b[1;32m     48\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39m\"\u001b[39m\u001b[39mstatus\u001b[39m\u001b[39m\"\u001b[39m \u001b[39min\u001b[39;00m query_result \u001b[39mand\u001b[39;00m query_result[\u001b[39m\"\u001b[39m\u001b[39mstatus\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39merror\u001b[39m\u001b[39m\"\u001b[39m:\n\u001b[1;32m     49\u001b[0m         \u001b[39mraise\u001b[39;00m requests\u001b[39m.\u001b[39mHTTPError(\n\u001b[1;32m     50\u001b[0m             \u001b[39m\"\u001b[39m\u001b[39mThe query returned an error, check if you mistyped an argument.\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m     51\u001b[0m         )\n",
      "File \u001b[0;32m~/Developement/py-star2bf/.direnv/python-3.10.10/lib/python3.10/site-packages/requests/models.py:975\u001b[0m, in \u001b[0;36mResponse.json\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m    971\u001b[0m     \u001b[39mreturn\u001b[39;00m complexjson\u001b[39m.\u001b[39mloads(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtext, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[1;32m    972\u001b[0m \u001b[39mexcept\u001b[39;00m JSONDecodeError \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m    973\u001b[0m     \u001b[39m# Catch JSON-related errors and raise as requests.JSONDecodeError\u001b[39;00m\n\u001b[1;32m    974\u001b[0m     \u001b[39m# This aliases json.JSONDecodeError and simplejson.JSONDecodeError\u001b[39;00m\n\u001b[0;32m--> 975\u001b[0m     \u001b[39mraise\u001b[39;00m RequestsJSONDecodeError(e\u001b[39m.\u001b[39mmsg, e\u001b[39m.\u001b[39mdoc, e\u001b[39m.\u001b[39mpos)\n",
      "\u001b[0;31mJSONDecodeError\u001b[0m: Expecting value: line 1 column 1 (char 0)"
     ]
    }
   ],
   "source": [
    "from reconciler import reconcile\n",
    "import pandas as pd\n",
    "\n",
    "# A DataFrame with a column you want to reconcile.\n",
    "test_df = pd.DataFrame(\n",
    "    {\n",
    "        \"City\": [\"Rio de Janeiro\", \"São Paulo\", \"São Paulo\", \"Natal\"],\n",
    "        \"Country\": [\"Q155\", \"Q155\", \"Q155\", \"Q155\"],\n",
    "        \"Land\": [\"XD-BR\", \"XD-BR\", \"XD-BR\", \"XD-BR\"]\n",
    "    }\n",
    ")\n",
    "\n",
    "funder_names = pd.DataFrame(\n",
    "    {\n",
    "        \"funder_name\": [ \"Bundesministerium für Bildung und Forschung (BMBF)\",\n",
    "                 \"Federal Ministry of Education and Research (BMBF)\",\n",
    "                 \"DFG\", \"Robert Bosch Foundation, Stuttgart, Germany\",\"Robert Bosch Foundation\",\"Robert Bosch Stiftung\",\n",
    "                 \"German Research Foundation, Clinical Research Unit 256\",\n",
    "                 \"German Research Society (DFG)\",\n",
    "                 \"German Research Society (Deutsche Forschungsgemeinschaft)\",\n",
    "                 \"DFG (German Research Foundation)\", \"German Research Society (Deutsche Forschungsgemeinschaft, DFG)\",\n",
    "                \"German Research Council\", \n",
    "               # \"Berlin University Alliance\",\n",
    "                \"Jacobs Foundation\",\n",
    "               # \"Typhaine Foundation\",\n",
    "               # \"European Commission\",\n",
    "                # \"JSPS Overseas Research Fellowship\",\n",
    "                \"German Research Society (DFG)\",\n",
    "              #  \"Villigst e.V.\",\n",
    "                \"Canada Research Chairs\",\n",
    "                \"Projekt DEAL\",\n",
    "                \"Natural Sciences and Engineering Research Council of Canada (NSERC)\",\n",
    "               # \"Templeton Religion Trust\",\n",
    "               # \"Austrian Science Fund (FWF)\",\n",
    "                \"Netherlands Organisation for Scientific Research\",\n",
    "                \"Advanced ERC grant\",\n",
    "               # \"Vertretungsnetz\",\n",
    "               # \"AOP Orphan\",\"Angelini\",\n",
    "              #  \"Science Foundation Ireland (SFI)\",\n",
    "              #  \"Interdisciplinary Center for Clinical Research (IZKF) of the medical faculty of Münster\"\n",
    "              ]\n",
    "    }\n",
    ")\n",
    "\n",
    "# Reconcile against type city (Q515), getting the best match for each item.\n",
    "reconciled = reconcile(funder_names[\"funder_name\"], reconciliation_endpoint=\"http://recon.labs.crossref.org/reconcile\")\n",
    "# reconciled = reconcile(funder_names[\"funder_name\"], type_id=\"TerritorialCorporateBodyOrAdministrativeUnit\", property_mapping={\"geographicAreaCode\": test_df[\"Land\"]}, reconciliation_endpoint=\"https://lobid.org/gnd/reconcile/\")\n",
    "# reconciled = reconcile(test_df[\"City\"], type_id=\"Q515\")\n",
    "\n",
    "# save the results to a csv file:\n",
    "test_df.to_csv(\"test.csv\")\n",
    "reconciled.to_csv(\"reconciled.csv\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# testing with some of our terms and GND Subject Headings:\n",
    "test_df = pd.DataFrame(\n",
    "     {\n",
    "         \"TermE\": [\"Inductive Deductive Reasoning\", \"Spatial Imagery\", \"Verbal Comprehension\"],\n",
    "         \"TermD\": [\"Induktiv-deduktives logisches Denken\", \"Räumliche Bildvorstellung\", \"Verbales Verständnis\"],\n",
    "     }\n",
    "    )\n",
    "\n",
    "reconciled = reconcile(test_df[\"TermE\"], type_id=\"SubjectHeading\", reconciliation_endpoint=\"https://lobid.org/gnd/reconcile/\")\n",
    "\n",
    "# save the results to a csv file:\n",
    "test_df.to_csv(\"test_terms.csv\")\n",
    "reconciled.to_csv(\"reconciled_terms.csv\")\n",
    "\n",
    "# This is not going to work, our cts just have too different wording compared to their gnd subject headings!\n",
    "\n",
    "\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use our authority records for insitutes to reconcile affiliations\n",
    "\n",
    "At first, I thought it was a good idea to do this by exposing our csv files as reconc apis using this JAVA tool: https://okfnlabs.org/reconcile-csv/\n",
    "\n",
    "It gives us a reconciliation API endpoint (on http://localhost:8000/reconcile) that one could use with the \"reconciler\" package, like this:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = pd.DataFrame(\n",
    "     {\n",
    "         \"Obstname\": [\"Äpfel\", \"Birne\", \"Himbeere\"],\n",
    "     }\n",
    "    )\n",
    "\n",
    "reconciled = reconcile(test_df[\"Obstname\"], reconciliation_endpoint=\"http://localhost:8000/reconcile\")\n",
    "reconciled.to_csv(\"reconciled_fruit.csv\")  \n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The above works, but we need to select which part to add to the record. It returns the label that was matched, a score, a match True/False (only True if score 1.0! Maybe we can lower the cutoff to 0.75?), and a type \n",
    "\n",
    "Also, this only works with dataframes, so whole tables, whereas we want to reconcile individual strings.\n",
    "\n",
    "Idea: \n",
    "Instead of reconciling with the API, we can just import the CSV of the authority institutes (as a list of dicts) and use fuzzywuzzy to match a given affiliation string. \n",
    "[Fuzzywuzzy](https://pypi.org/project/fuzzywuzzy/) (\"Fuzzy string matching in python\") compares two strings and returns a score. We can use a cutoff of 75% to decide if the string matches the label.\n",
    "\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import our authority institutes as CSV, use fuzzywuzzy to match a given affiliation string to them\n",
    "\n",
    "Fuzzywuzzy, if passed a list with sublists for synonyms, also automatically looks in there. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# from rapidfuzz import fuzz\n",
    "# from rapidfuzz import process\n",
    "import Levenshtein\n",
    "from rapidfuzz.process import extractOne\n",
    "from rapidfuzz.fuzz import ratio\n",
    "from rapidfuzz.fuzz import token_set_ratio\n",
    "import csv\n",
    "\n",
    "# import csv file with dachlux institutes:\n",
    "with open('institute_lux.csv', newline='') as csvfile:\n",
    "    reader = csv.DictReader(csvfile)\n",
    "    # save it in a list:\n",
    "    dachlux_institutes = list(reader)\n",
    "    # split string \"known_names\" into a list of strings on \"##\":\n",
    "    for institute in dachlux_institutes:\n",
    "        institute[\"known_names\"] = institute[\"known_names\"].split(\" ## \")\n",
    "# print(\"Und die ganze Tabelle:\")\n",
    "# print(dachlux_institutes)\n",
    "\n",
    "\n",
    "# affiliation_string = \"Abteilung für Psychologie, University of Luxembourg, Esch-sur-Alzette\"\n",
    "# expected match: uuid: bfe28ac0-4901-4125-aaa6-c1fb6c644b7a, Centre de Prévention des Toxicomanies (CePT)\n",
    "\n",
    "def match_local_authority_institutes(string, list):\n",
    "    # this function takes a string and returns the best match from a list of strings\n",
    "    # first, get the list of strings:\n",
    "    # then, get the best match (token_set_ratio seems to be the best scorer for our purposes, so we use that):)):\n",
    "    # It yields 100% for exact matches. It is also insensitive to word order differences.\n",
    "    # best_match = process.extractOne(string, list, scorer=fuzz.token_set_ratio)\n",
    "    # best_match = extractOne(string, list, scorer=token_set_ratio)\n",
    "    return extractOne(string, list, scorer=token_set_ratio)\n",
    "    # return best_match\n",
    "    # or if i just wanted the value in \"uuid\":\n",
    "    # return best_match[0].get(\"uuid\")\n",
    "\n",
    "hundeliste = (\"Hundefriseur\", \"Hundefrise\", \"Hundefrisur\")\n",
    "\n",
    "match_local_authority_institutes(\"Frosch\", dachlux_institutes[list==[\"known_names\"]])\n",
    "# print(\"Am besten passt:\",match_local_authority_institutes(\"Fakultät für Geisteswissenschaften, Erziehungswissenschaften und Sozialwissenschaften; Universität Luxemburg\", dachlux_institutes))\n",
    "# print(\"Am besten passt:\",match_local_authority_institutes(\"Department of Behavioral and Cognitive Sciences, University of Luxembourg, Esch-sur-Alzette\", dachlux_institutes))\n",
    "\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To use this in the script, we need the following pseudo-code:\n",
    "\n",
    "- check if the affiliation string contains a country name\n",
    "- if yes, check if the country name is in the list of countries (D, A, CH, LUX)\n",
    "- only then use fuzzywuzzy to match the affiliation string to the list of institutes for that country\n",
    "\n",
    "If we have separate csv files for each country, we can use the country name to select the right file.\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use a ror id that we have to look up more data\n",
    "\n",
    "For authority records, we can use the ror id (that we already looked up using the api with the affiliation parameter) to look up more data about the institute, like the country, the name, the aliases, etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Centre for European Economic Research', 'ZEW', 'Mannheim', 2873891, 'DE', 'Germany', {'ISNI': {'preferred': None, 'all': ['0000 0004 0492 4665']}, 'Wikidata': {'preferred': None, 'all': ['Q191206']}, 'GRID': {'preferred': 'grid.13414.33', 'all': 'grid.13414.33'}})\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# https://api.ror.org/organizations/00tjv0s33\n",
    "\n",
    "# for getting data about a known id:\n",
    "import requests_cache\n",
    "from datetime import timedelta\n",
    "\n",
    "ROR_API_LOOKUP_URL = \"https://api.ror.org/organizations/\"\n",
    "\n",
    "urls_expire_after = {\n",
    "}\n",
    "\n",
    "session_ror = requests_cache.CachedSession(\n",
    "    \".cache/requests\",\n",
    "    allowable_codes=[200, 404],\n",
    "    expire_after=timedelta(days=30),\n",
    "    urls_expire_after=urls_expire_after,\n",
    ")\n",
    "\n",
    "# For research institutes/departments, it makes sense to use ror-id for things that are true for the parent \n",
    "# organization as well as for the department: country code, the country name, the city. \n",
    "\n",
    "# Trouble is: we have no way to know if a reconciled ror-id is for a department or for its parent organization.\n",
    "# if sb used their university as the affiliation, we would get the ror-id for the university, \n",
    "# but if sb has a department, we still get the ror-id for the university, and we can't say: \n",
    "# this is the ror-id for the parent or for this exact suborg. Maybe! There is a way to find out: \n",
    "# if it's a full match (not partial) then it is a \"sameAs\" relationship,\n",
    "# if it's only a partial match to the org name, it's a \"related\" or \"Child\" relationship?.\n",
    "\n",
    "# The response is a JSON object containing a full ROR record. \n",
    "# See [ROR data structure](https://ror.readme.io/docs/ror-data-structure) for details about the fields and values in a ROR record.\n",
    "# we are interested in:\n",
    "# acryonyms (list)\n",
    "# aliases (list)\n",
    "# country.country_code (string), country.country_name (string)\n",
    "# addresses[0].city (string), \n",
    "# addresses[0].country_geonames_id -> can link to our own geographica's\n",
    "# addresses[0].geonames_city.id (string), maybe: addresses[0].geonames_city.name (string)\n",
    "# Allowed external IDs: \n",
    "#  Funder ID (FundRef), ISNI, Wikidata. Other external IDs not actively curated include GRID, OrgRef, HESA, UCAS, UKPRN, CNRS.\n",
    "# external_ids.ISNI.preferred (string), or if \"null\": .external_ids.ISNI.all (array, or just use the first [0])\n",
    "# external_ids.Wikidata.preferred (string), or if \"null\": .external_ids.Wikidata.all (array, or just use the first [0])\n",
    "# relationships (array) mit jeweils:\n",
    "#    .id (ror-id mit http-Vorspann der related org)\n",
    "#    .type (string, eins von \"Related\", \"Successor\",\"Predecessor\", \"Parent\", \"Child\")\n",
    "#    .label (string, z.B. \"Leibniz-Association\")\n",
    "# we also want the German name or name in other languages, if available. \n",
    "# it should be in .labels : labels[0].iso639 = \"de\", labels[0].label = \"Leibniz-Gemeinschaft\"\n",
    "# and we can construct a langstring from it like \"Leibniz-Gemeinschaft\"@de\n",
    "\n",
    "\n",
    "def get_ror_authority_data(ror_id):\n",
    "    ror_api_url = ROR_API_LOOKUP_URL + ror_id\n",
    "    # make a request to the ror api:\n",
    "    # ror_api_request = requests.get(ror_api_url)\n",
    "    # make request to api with caching:\n",
    "\n",
    "    # put in a try/except block to catch timeouts:\n",
    "    try:\n",
    "        ror_api_request = session_ror.get(\n",
    "            ror_api_url, timeout=20\n",
    "        )\n",
    "    except TimeoutError:\n",
    "        print(\"Timeout!\")\n",
    "        return None\n",
    "    else: \n",
    "        # if the request was successful, get the json response:\n",
    "        if ror_api_request.status_code == 200:\n",
    "            try:\n",
    "                ror_api_response = ror_api_request.json()\n",
    "            except:\n",
    "                print(\"Error getting json response!\")\n",
    "                return None\n",
    "            # if we got the response (try ran successfully), \n",
    "            # do something with it:\n",
    "            else:\n",
    "                try:\n",
    "                    name = ror_api_response[\"name\"]\n",
    "                except:\n",
    "                    name = None\n",
    "                    print(\"Error getting name!\")\n",
    "                for acronym in ror_api_response[\"acronyms\"]:\n",
    "                    if acronym is not None:\n",
    "                        acronym = acronym\n",
    "                    else:\n",
    "                        acronym = None\n",
    "                city = ror_api_response[\"addresses\"][0][\"city\"]\n",
    "                geonames_city = ror_api_response[\"addresses\"][0][\"geonames_city\"][\"id\"]\n",
    "                country_code = ror_api_response[\"country\"][\"country_code\"]\n",
    "                country_name = ror_api_response[\"country\"][\"country_name\"]\n",
    "                external_ids = ror_api_response[\"external_ids\"]\n",
    "\n",
    "            return name,acronym, city, geonames_city, country_code, country_name, external_ids\n",
    "        else:\n",
    "            return None\n",
    "    \n",
    "\n",
    "# print(get_ror_authority_data(\"0165gz615\")) # ZPID\n",
    "print(get_ror_authority_data(\"02qnsw591\"))\n",
    "print(get_ror_authority_data(\"random1\")) "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feld GRANT migrieren\n",
    "\n",
    "- Vor allem Unterfeld |n - auftrennen! |i aufheben, und wenn möglich überflüssiges, was keine Nummer ist, wegwerfen.\n",
    "\n",
    "\n",
    "Ergebnis soll so aussehen (ähnlich wie bei Crossref:, aber mit grant_name ähnoich DataCite, dort heißt es aber grantTitle ) :\n",
    "\n",
    "```\n",
    "{\n",
    "    'funder': \n",
    "    {\n",
    "        'funder_name': 'Sächsische Aufbaubank ^DDS Förder bank ^DDS (SAB)', 'funder_id': None\n",
    "    }, \n",
    "        'grants': \n",
    "        [\n",
    "            {\n",
    "                'grant_number': '100362999 an YG', \n",
    "                'grant_name': None\n",
    "            }\n",
    "        ], \n",
    "        'funding_note': None\n",
    "},\n",
    "{'funder': {'funder_name': 'Institute for Applied Research, Development and Further Education (IAF) at the Catholic University of Applied Sciences in Freiburg', 'funder_id': None}, 'grants': None, 'funding_note': None},\n",
    "{\n",
    "    'funder': \n",
    "    {\n",
    "        'funder_name': 'JSPS KAKENHI', \n",
    "        'funder_id': None\n",
    "    }, \n",
    "    'grants': [\n",
    "        {\n",
    "            'grant_number': '15K00871', \n",
    "            'grant_name': None\n",
    "        }, \n",
    "        {\n",
    "            'grant_number': '18KK0055', \n",
    "            'grant_name': None\n",
    "        }\n",
    "    ], \n",
    "    'funding_note': None}\n",
    "```\n",
    "\n",
    "Note: may remove the general \"funding_note\" field, anywhere else, or move its contents as grant_name to each grant (as OpenAlex does). May rename "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_strings = (\"KND1: 01GI0102, 01GI0420, 01GI0422, 01GI0423, 01GI0429, 01GI0431, 01GI0433, 01GI0434; KNDD: 01GI0710, 01GI0711, 01GI0712, 01GI0713, 01GI0714, 01GI0715, 01GI0716\",\n",
    "             \"HO5852/1-1\",\"392443797\",\"01GI1008C\",\"801210010-20\",\n",
    "             \"TA 857/3-2\", \"2016YFC1306800\", \"81671329\", \"18ZDA293\", \"17411969900\",\n",
    "             \"20144Y0053\", \"SHDC12014111\", \"13dz2260500\", \"ZH2018QNB19\",\n",
    "             \"2018-FX-04, 2013-YJGJ-03\", \"IIR-1303\", \n",
    "             \"01GL1714A; 01GL1714B; 01GL1714C; 01GL1714D, 01GY1613\",\n",
    "             \"15K00871 and 18KK0055\", \"366/14\", \"386/14\", \"100362999 an YG\"\n",
    "             )\n",
    "\n",
    "import rdflib\n",
    "\n",
    "\n",
    "\n",
    "def extract_grant_numbers(subfield_n_string):\n",
    "    # this function takes a string and returns a list of award numbers\n",
    "    # first, split the string on \",\" or \";\" or \"and\": (first replacing all semicolons and \"ands\" with commas)\")\n",
    "    subfield_n_string = subfield_n_string.replace(\" and \", \", \")\n",
    "    subfield_n_string = subfield_n_string.replace(\";\", \",\")\n",
    "    subfield_n_string = subfield_n_string.split(\", \")\n",
    "    # in each of the returned list elements, remove any substrings that are shorter \n",
    "    # than 5 characters (to get rid of things like \" for\" or \"KDL: \" YG: \" etc.)\n",
    "    # for element in subfield_n_string:\n",
    "    #     if len(element) < 5:\n",
    "    #         subfield_n_string.remove(element)\n",
    "    # go through all the list elements and replace each with a dict,\n",
    "    # which has a key \"grant_number\" and a key \"grant_name\" (which is None for now):\n",
    "    for i, element in enumerate(subfield_n_string):\n",
    "        subfield_n_string[i] = {\"grant_number\": element, \"grant_name\": None}\n",
    "    # return the list of dicts:\n",
    "    return subfield_n_string\n",
    "\n",
    "# extract_grant_numbers(n_strings[0])\n",
    "\n",
    "\n",
    "\n",
    "def build_grant_from_starfield(grantfield):\n",
    "    # this function takes a string and returns a funder, grant number, grant name, grant holder\n",
    "    # first, use anything before the first \"|\" as the funder:\n",
    "    funder = {\"funder_name\": grantfield.split(\"|\")[0].strip(), \"funder_id\": None}\n",
    "    # then check the rest for a grant number:\n",
    "    if \"|n \" in grantfield:\n",
    "        grants = grantfield.split(\"|n \")[1].split(\" |\")[0]\n",
    "        grants = extract_grant_numbers(grants)\n",
    "    else:\n",
    "        grants = None\n",
    "    # then check the rest for a grant name:\n",
    "    if \"|i \" in grantfield:\n",
    "        funding_info = grantfield.split(\"|i \")[1].split(\" |\")[0]\n",
    "    else:\n",
    "        funding_info = None\n",
    "    if \"|e \" in grantfield:\n",
    "        funding_recipients = grantfield.split(\"|e \")[1].split(\" |\")[0]\n",
    "        if funding_info is not None:\n",
    "            funding_info = funding_info + \". Recipient(s): \" + funding_recipients\n",
    "        else:\n",
    "            funding_info = \"Recipient(s): \" + funding_recipients\n",
    "    # return a dict of the variables:\n",
    "    return {\"funder\": funder, \"grants\": grants, \"funding_note\": funding_info}\n",
    "\n",
    "GRANTs = (\n",
    "\"Deutsche Forschungsgemeinschaft |e L.M. |i \\\"Pragmatic Functions and Effects of Register Variation and Switch: a Register approach to negation and polarity\\\" (SFB 1412 \\\"Register\\\"; project number: 416591334)\"\n",
    ")\n",
    "\n",
    "# print(build_grant_from_starfield(GRANTs[3]))\n",
    "\n",
    "#for grant in GRANTs:\n",
    " #    pass\n",
    "    #print(build_grant_from_starfield(grant))\n",
    "\n",
    "print(build_grant_from_starfield(GRANTs))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('something&amp;Else', 'Funder Not found')\n",
      "\n",
      "('Bundesministerium für Bildung und Forschung (BMBF)', '10.23456/501100002347')\n",
      "\n",
      "('Federal Ministry of Education and Research (BMBF)', '10.23456/501100002347')\n",
      "\n",
      "('Deutsche Forschungsgemeinschaft (DFG)', '10.23456/501100001659')\n",
      "\n",
      "('Robert Bosch Foundation, Stuttgart, Germany', 'Funder Not found')\n",
      "\n",
      "('Robert Bosch Foundation', '10.23456/501100001646')\n",
      "\n",
      "('Robert Bosch Stiftung', '10.23456/501100001646')\n",
      "\n",
      "('German Research Foundation, Clinical Research Unit 256', 'Funder Not found')\n",
      "\n",
      "('Deutsche Forschungsgemeinschaft (DFG)', '10.23456/501100001659')\n",
      "\n",
      "('Deutsche Forschungsgemeinschaft (DFG)', '10.23456/501100001659')\n",
      "\n",
      "('DFG (German Research Foundation)', '10.23456/501100001659')\n",
      "\n",
      "('Deutsche Forschungsgemeinschaft (DFG)', '10.23456/501100001659')\n",
      "\n",
      "('Deutsche Forschungsgemeinschaft (DFG)', '10.23456/501100001659')\n",
      "\n",
      "('Jacobs Foundation', '10.23456/501100006301')\n",
      "\n",
      "('Deutsche Forschungsgemeinschaft (DFG)', '10.23456/501100001659')\n",
      "\n",
      "('Canada Research Chairs', '10.23456/501100002784')\n",
      "\n",
      "Skipping Projekt DEAL\n",
      "None\n",
      "\n",
      "('Natural Sciences and Engineering Research Council of Canada (NSERC)', '10.23456/501100000038')\n",
      "\n",
      "('Netherlands Organisation for Scientific Research', '10.23456/501100019926')\n",
      "\n",
      "('Advanced ERC grant', 'Funder Not found')\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from urllib.parse import urlencode\n",
    "import requests_cache\n",
    "from datetime import timedelta\n",
    "from modules.mappings import dd_codes\n",
    "from modules.mappings import funder_names_replacelist\n",
    "import html\n",
    "# from mappings import geonames_countries\n",
    "# from mappings import abstract_origins\n",
    "\n",
    "skip_these_grants = (\n",
    "    \"projekt deal\", \"project deal\", \"open access funding\"\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "# set up friendly session by adding mail in request:\n",
    "CROSSREF_FRIENDLY_MAIL = \"&mailto=ttr@leibniz-psychology.org\"\n",
    "# for getting a list of funders from api ():\n",
    "CROSSREF_API_URL = \"https://api.crossref.org/funders?query=\"\n",
    "\n",
    "\n",
    "def replace_encodings(text):\n",
    "    for case in dd_codes:\n",
    "        text = text.replace(case[0], case[1]) \n",
    "    return text\n",
    "\n",
    "def replace_common_fundernames(funder_name):\n",
    "    \"\"\"This will accept a funder name that crossref api may not recognize, at least not as the first hit,\n",
    "    and replace it with a string that will supply the right funder as the first hit\"\"\"\n",
    "    # if the funder_name is in the list of funder names to replace (in index 0), then replace it with what is in index 1:\n",
    "    for funder in funder_names_replacelist:\n",
    "        if funder_name == funder[0]:\n",
    "            funder_name = funder[1]\n",
    "    return funder_name\n",
    "    \n",
    "urls_expire_after = {\n",
    "    # Custom cache duration per url, 0 means \"don't cache\"\n",
    "    # f'{SKOSMOS_URL}/rest/v1/label?uri=https%3A//w3id.org/zpid/vocabs/terms/09183&lang=de': 0,\n",
    "    # f'{SKOSMOS_URL}/rest/v1/label?uri=https%3A//w3id.org/zpid/vocabs/terms/': 0,\n",
    "}\n",
    "\n",
    "session = requests_cache.CachedSession(\n",
    "    \".cache/requests\",\n",
    "    allowable_codes=[200, 404],\n",
    "    expire_after=timedelta(days=30),\n",
    "    urls_expire_after=urls_expire_after,\n",
    ")\n",
    "\n",
    "\n",
    "def get_crossref_funder_id(funder_name):\n",
    "    # this function takes a funder name and returns the crossref funder id for that funder name\n",
    "    # to do this, use the crossref api.\n",
    "    if funder_name.lower() in skip_these_grants:\n",
    "        print(\"Skipping \" + funder_name)\n",
    "    else:\n",
    "        funder_name = replace_common_fundernames(funder_name)\n",
    "        # encode for url parameters (that is, remove any html entities with an & in front of them):\n",
    "        #funder_name = html.unescape(funder_name)\n",
    "        # construct the api url:\n",
    "        crossref_api_url = CROSSREF_API_URL + funder_name + CROSSREF_FRIENDLY_MAIL\n",
    "        # + CROSSREF_FRIENDLY_MAIL\n",
    "        # make a request to the crossref api:\n",
    "        # crossref_api_request = requests.get(crossref_api_url)\n",
    "        # make request to api:\n",
    "        try:\n",
    "            crossref_api_request = session.get(\n",
    "                crossref_api_url, timeout=20\n",
    "            )\n",
    "        except TimeoutError:\n",
    "            print(\"Timeout!\")\n",
    "            return None\n",
    "        else:\n",
    "            # if the request was successful, get the json response:\n",
    "            crossref_api_response = crossref_api_request.json()\n",
    "            if crossref_api_request.status_code == 200 and crossref_api_response[\"message\"][\"total-results\"] >=1:\n",
    "                first_hit = f'10.23456/{crossref_api_response[\"message\"][\"items\"][0][\"id\"]}'\n",
    "            else:\n",
    "                first_hit = \"Funder Not found\"\n",
    "            return funder_name, first_hit\n",
    "\n",
    "\n",
    "funderstrings = (\"something&amp;Else\",\n",
    "                 \"Bundesministerium für Bildung und Forschung (BMBF)\",\n",
    "                 \"Federal Ministry of Education and Research (BMBF)\",\n",
    "                 \"DFG\", \"Robert Bosch Foundation, Stuttgart, Germany\",\"Robert Bosch Foundation\",\"Robert Bosch Stiftung\",\n",
    "                 \"German Research Foundation, Clinical Research Unit 256\",\n",
    "                 \"German Research Society (DFG)\",\n",
    "                 \"German Research Society (Deutsche Forschungsgemeinschaft)\",\n",
    "                 \"DFG (German Research Foundation)\", \"German Research Society (Deutsche Forschungsgemeinschaft, DFG)\",\n",
    "                \"German Research Council\", \n",
    "               # \"Berlin University Alliance\",\n",
    "                \"Jacobs Foundation\",\n",
    "               # \"Typhaine Foundation\",\n",
    "               # \"European Commission\",\n",
    "                # \"JSPS Overseas Research Fellowship\",\n",
    "                \"German Research Society (DFG)\",\n",
    "              #  \"Villigst e.V.\",\n",
    "                \"Canada Research Chairs\",\n",
    "                \"Projekt DEAL\",\n",
    "                \"Natural Sciences and Engineering Research Council of Canada (NSERC)\",\n",
    "               # \"Templeton Religion Trust\",\n",
    "               # \"Austrian Science Fund (FWF)\",\n",
    "                \"Netherlands Organisation for Scientific Research\",\n",
    "                \"Advanced ERC grant\",\n",
    "               # \"Vertretungsnetz\",\n",
    "               # \"AOP Orphan\",\"Angelini\",\n",
    "              #  \"Science Foundation Ireland (SFI)\",\n",
    "              #  \"Interdisciplinary Center for Clinical Research (IZKF) of the medical faculty of Münster\"\n",
    "                )\n",
    "\n",
    "# import rdflib\n",
    "# from rdflib import Graph, Literal, RDF, URIRef, Namespace\n",
    "# from rdflib.namespace import SKOS, DC, DCTERMS, FOAF, OWL, RDF, RDFS, XSD\n",
    "\n",
    "# # new namespace skosxl:\n",
    "# SKOSXL = Namespace(\"http://www.w3.org/2008/05/skos-xl#\")\n",
    "\n",
    "# fundref_registry = Graph()\n",
    "# fundref_registry.parse(\"crossref_fundref_registry.rdf\", format=\"xml\")   \n",
    "\n",
    "# def crossref_local_lookup(funder_name):\n",
    "#     # if the name is a skosxl:prefLabel/skosxl:Label/skosxl:literalForm in the fundref registry, then return the fundref id:\n",
    "#     # first, check if the funder_name is a skosxl:prefLabel/skosxl:Label/skosxl:literalForm in the fundref registry:\n",
    "#     # if it is, return the fundref id:\n",
    "#     for s, p, o in fundref_registry.triples((None, SKOSXL.prefLabel, Literal(funder_name))):\n",
    "#         print(\"Found \" + funder_name + \" in fundref registry as skosxl:prefLabel\")\n",
    "#         # print(\"Fundref id: \" + s)\n",
    "#         # return s\n",
    "\n",
    "# for funder in funderstrings:\n",
    "#     crossref_local_lookup(funder)\n",
    "#     #print(funder)\n",
    "\n",
    "for funder in funderstrings:\n",
    "    #print(\"Funder: \" + funder)\n",
    "    print(get_crossref_funder_id(funder))\n",
    "    print(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Getting conference info and transforming into a contribution\n",
    "\n",
    "Currently in field CF. However, we have over 300 publications with CF that can be dropped. We only want to keep and transform the field for BE types SS and SM (books, either edited or authored).\n",
    "\n",
    "We transform the content of the CF field and its subfields into a bf:Contribution, where the agent is a BF:Meeting, which conforms to RDA and Bibframe.\n",
    "- Required: one conference name \n",
    "- Optional: one year (extracted from heterogeneous date strings in subfield |d)\n",
    "- Optional: one place as a Literal/string (extracted from subfield |o). There will be no matching with geonames or our own place/cities authority, because there is no real use case for it, and because the data is too heterogeneous (sometimes a building in a city, sometimes a hosting institution, such as a university).\n",
    "- Optional: one info field, extracted from subfield |b. This will also be used to hold any leftover complex dates from subfield |d.\n",
    "\n",
    "Bibframe example:\n",
    "\n",
    "```r\n",
    "<ProceedingsWork> a bf:Work ;\n",
    "    bf:contribution [a pxc:ConferenceReference ; # a bf:Contribution ;\n",
    "        bf:agent [a bf:Meeting ;\n",
    "            rdfs:label \"Tagung der Arbeitsgemeinschaft Psychodynamischer Professorinnen und Professoren\" ;\n",
    "            bf:identifiedBy [a pxc:ConferenceDoi ;\n",
    "                rdf:value \"10.12344\" ; \n",
    "                bf:source \"TIB\"; # oder so\n",
    "            ] ;\n",
    "            bflc:simplePlace \"Berlin\" ;\n",
    "            bflc:simpleDate \"2019\"^^xsd:gYear ; \n",
    "        ] ;\n",
    "        bf:role <http://id.loc.gov/vocabulary/relators/ctb> ;\n",
    "        bf:note [a bf:Note ;\n",
    "            rdfs:label \"Date: 03.-04.10.2019, International Psychoanalytic University (IPU) Berlin\" ;\n",
    "        ] ;\n",
    "    ]\n",
    "    .\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'conference_name': 'Tagung der Group Analytic Society International (GASi)', 'conference_pid': None, 'year': '2017', 'location': 'Berlin', 'conference_note': 'Date(s): 2017'}\n",
      "{'conference_name': 'DVG-Fachtagung', 'conference_pid': None, 'year': None, 'location': 'Frankfurt a. M.', 'conference_note': None}\n",
      "{'conference_name': 'Tagung der Arbeitsgemeinschaft Psychodynamischer Professorinnen und Professoren', 'conference_pid': None, 'year': '2019', 'location': None, 'conference_note': 'Date(s): 03.-04.10.2019. International Psychoanalytic University (IPU) Berlin'}\n"
     ]
    }
   ],
   "source": [
    "# sample strings from CF field:\n",
    "cf_strings = {\n",
    "    'Tagung der Group Analytic Society International (GASi) |d 2017 |o Berlin',\n",
    "    'DVG-Fachtagung |o Frankfurt a. M. |b Vortrag \"Krise! Welche Krise? Oder: Lernen ist immer eine Möglichkeit\"',\n",
    "    'Tagung der Arbeitsgemeinschaft Psychodynamischer Professorinnen und Professoren |d 03.-04.10.2019 |b International Psychoanalytic University (IPU) Berlin',\n",
    "}\n",
    "\n",
    "import re\n",
    "\n",
    "def make_conference_node(string):\n",
    "    # this function takes a string and returns a conference node\n",
    "    # initialize the variables with None:\n",
    "    conference_name = None\n",
    "    conference_pid = None\n",
    "    contribution_role = \"conference\"\n",
    "    date = None\n",
    "    year = None\n",
    "    location = None\n",
    "    conference_note = None\n",
    "    # first, use the first part before the first \"|\" as the conference name:\n",
    "    conference_name = string.split(\"|\")[0].strip()\n",
    "    # then check the rest for a date:\n",
    "    try:\n",
    "        date = string.split(\"|d \")[1].split(\" |\")[0]\n",
    "    except:\n",
    "        date = None\n",
    "    else:\n",
    "        # copy the date into conference_note:\n",
    "        conference_note = \"Date(s): \" + date\n",
    "        # check date for a year: anything with 4 digits anywhere in the date string is a year:\n",
    "        # use a regex for finding YYYY pattern in any string:\n",
    "        year_pattern = re.compile(r\"\\d{4}\")\n",
    "        # if there is a year in the date string, use that as the date:\n",
    "        if year_pattern.search(date):\n",
    "            year = year_pattern.search(date).group()\n",
    "        else:\n",
    "            year = None\n",
    "    # then check the rest for a location:\n",
    "    try:\n",
    "        location = string.split(\"|o \")[1].split(\" |\")[0]\n",
    "    except:\n",
    "        location = None\n",
    "    # then check the rest for a conference note:\n",
    "    try:\n",
    "        conference_note = conference_note + \". \" + string.split(\"|b \")[1]\n",
    "    except:\n",
    "        conference_note = conference_note\n",
    "    # return a dict of the variables:\n",
    "    return {\"conference_name\": conference_name, \"conference_pid\": conference_pid, \"year\": year, \"location\": location, \"conference_note\": conference_note}\n",
    "\n",
    "for cf_string in cf_strings:\n",
    "    print(make_conference_node(cf_string))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dealing with Bibliographic notes in field BN\n",
    "\n",
    "## Splitting BN: \"Original: \" entries\n",
    "\n",
    "We can use what is in BN of a translation to create relationships to the original publication - by splitting it into two fields, the title with edition and the provision activity statement string (place, publisher, year). This is useful for books.\n",
    "\n",
    "So we can make do with two string fields for the new \"related work\" stuff - for originals of translations, at least!\n",
    "\n",
    "Bibframe example:\n",
    "\n",
    "```r\n",
    "<TranslatedBookWork> a bf:Work ;\n",
    "    # bf:title [a bf:Title ; rdfs:label \"Qualitative Forschung.\" ] ; # actually, we should export the original title here, too, because it is the \"real\"/preferred title of the work. But do we have to? \n",
    "    bflc:relationship [a bflc:Relationship ; \n",
    "        bflc:relation relations:hasTranslation ;\n",
    "        bf:translationOf [\n",
    "            a bf:Work ; \n",
    "            bf:hasInstance [\n",
    "                a bf:Instance ;\n",
    "\t                bf:title [a bf:Title ; rdfs:label \"Qualitative Forschung. 3. überarb. Aufl.\" ] ; # we could try to find the language? \n",
    "\t                bf:provisionActivityStatement \"Reinbek: Rowohlt Taschenbuch Verlag, 1995\" .\n",
    "            ]\n",
    "        ]\n",
    "    ]\n",
    "<Instance> \n",
    "```\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "original_strings = {\n",
    "    \"Original: 2008. Psychische stoornissen, gedragsproblemen en verstandelijke handicap. Een integratieve benadering voor kinderen en volwassenen. 3., bearb. Aufl. Assen: Van Gorcum\",\n",
    "    \"Original: 2008. Feeling good together. The secret to making troubled relationships work. New York: Broadway Books\",\n",
    "    \"Original: 1976. L'hystérique, le sexe et le médecin. Paris: Masson\" ,\n",
    "    \"Original: 198O. Mindstorms. Children, Computer, and powerful ideas. New York: Basic Books\" , # note: year has a letter!\n",
    "    \"Original: 1978. The child and his symptoms. Third edition. Oxford: Blackwell\" ,\n",
    "    \"Original: 1975. Vys#s27aja nervnaja dejatel'nost' #c27eloveka motivacionno-emocional'nye aspekty. Moskau: Izdatel'stvo Nauka\" ,\n",
    "    # outliers:\n",
    "    \"Original: 1981. Crescere. Roma: Astrolabio-Ubaldini.\" , # dot at the end\n",
    "    \"Original. 2012. Doing Dialectical Behavior Therapy: A practical guide. New York: Guilford Press\" , # dot instead of : after Original\n",
    "    \"Original. 1975. Psychology. Boston: Little, Brown and Company\" , # same,\n",
    "    \"Deutschsprachiges Original: 2006. Motivation und Handeln. - 3., überarb. u. aktualis. Aufl. Berlin: Springer\" , # has \"Deutschsprachiges\" at the start\n",
    "    \"Englische Übersetzung des Originals. 1995. Qualitative Forschung. Reinbek: Rowohlt Taschenbuch Verlag\" , # has \"Englische Übersetzung des Originals\" at the start, dot instead of : after Original\n",
    "    \"Englische Übersetzung des Originals: 1989. Beziehungen und Probleme verstehen. Eine Einführung in die psychotherapeutische Plananalyse. Bern: Huber\" , # same, bit has a colon after Original\n",
    "    \"Englische Übersetzung des deutschsprachigen Originals: 1998. Namenlos. Geistig Behinderte verstehen. - 3., überarb. Aufl.- Neuwied: Luchterhand\", # same, but has \"Englische Übersetzung des deutschsprachigen Originals\" at the start and \"-\" instead of \".\" before the place\n",
    "}\n",
    "\n",
    "# - [ ] recognize language?!\n",
    "\n",
    "import langid\n",
    "langid.set_languages([\"de\", \"en\", \"nl\", \"fr\", \"it\", \"ru\"])\n",
    "\n",
    "def guess_language(string_in_language):\n",
    "    return (langid.classify(string_in_language)[0])\n",
    "\n",
    "def split_bn_original(string):\n",
    "    # this function takes a string and returns a dict with the original string and the bn string\n",
    "    # first, remove the \"Original: \" from the string, if it exists - it is always at the start:\n",
    "    # these are its variations:\n",
    "    prefix_variations = [\"Original: \", \"Original. \", \"Deutschsprachiges Original: \", \n",
    "                         \"Englische Übersetzung des Originals. \", \"Englische Übersetzung des Originals: \", \n",
    "                         \"Englische Übersetzung des deutschsprachigen Originals: \" ]\n",
    "    # if string.startswith(\"Original: \") or string.startswith(\"Original. \"):\n",
    "    # check if the string starts with any of the variations:\n",
    "        # redefine the string as the part after \"Original: \" (which is as long as the \"original\" prefix):\n",
    "        # get the variation that is at the start of the string:\n",
    "    for variation in prefix_variations:\n",
    "        if string.startswith(variation):\n",
    "        # redefine the string as the part after the variation:\n",
    "            string = string[len(variation):]\n",
    "            # then split out the year into a separate variable. It always comes first, has four characters, and is followed by a dot:\n",
    "            year = string[:4]\n",
    "            # if there is a O in the year, replace it with a 0:\n",
    "            if \"O\" in year:\n",
    "                year = year.replace(\"O\", \"0\")\n",
    "            # then split out the provision activity (place and publisher) into a separate variable. It always comes last, is preceded by a dot and has a colon between place and publisher:\n",
    "            provision_activity = string.split(\". \")[-1]\n",
    "            # if it ends in a dot, remove it:\n",
    "            if provision_activity.endswith(\".\"):\n",
    "                provision_activity = provision_activity[:-1]\n",
    "            # then split the title and edition info into another variable. It is everything between the year and the provision activity. Keep any dots in the title, as they are part of the title, but strip any whitespace at the start or end:\n",
    "            title = string[5:-len(provision_activity)-1].strip()\n",
    "            # guess the language of the title:\n",
    "            title_language = guess_language(title)\n",
    "            # add content of year to the provision activity in a new variable provision_activity_statement:\n",
    "            provision_activity_statement = provision_activity + \", \" + year\n",
    "            # print as a bibframe Instance with title and provision activity, but use f-strings:\n",
    "            print(f\"<Instance> a bf:Instance ;\\n\\tbf:title [a bf:Title ;\\n\\trdfs:label \\\"{title}\\\" {title_language} ] ;\\n\\tbf:provisionActivityStatement \\\"{provision_activity_statement}\\\" .\")\n",
    "            # return {\"title\": title, \"provision_activity_statement\": provision_activity_statement}\n",
    "\n",
    "for counter, element in enumerate(original_strings):\n",
    "    # print(split_bn_original(element))\n",
    "    print(counter)\n",
    "    split_bn_original(element)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BN with other things:\n",
    "\n",
    "Simple statements that are looking for their own field or vocabulary:\n",
    "- \"Offsetdruck\" - 2241x - Hoschschulschrift gedruckt, also eher mehrere Exemplare. (wohin damit?) \n",
    "- \"Schreibmaschinenfasung\" (2858x!) - bedeutet: ist eine Hochschulschrift, die eher ein Manuskript ist, also nur wenie Exemplare vorliegen. bf:Manuscript zuordnen???\n",
    "- Microfichefassung (52x)\n",
    "- Loseblattsammlung (8x), \"Loseblattausgabe\", \"Loseblattausgabe im Ordner\", \"\"Loseblattausgabe im Ringbuchordner\", \"Loseblattausgabe in Ordner\", \n",
    "- \"Kumulative Dissertation\" (3000x), \n",
    "    - \"Kumulative Dissertation, bestehend aus mehreren Buch- und Zeitschriftenbeiträgen\", \n",
    "    - \"Kumulative Dissertation: (1) Pavel, F.-G. 1978. Die klientenzentrierte Psychotherapie. München: Pfeiffer; (2) Beitrag in: Spiel, W. (Ed.) 1980. Die Psychologie des 20. Jahrhunderts. Bd. 12. Zürich: Kindler. S. 844-864; (3) GwG-Info 1982, 47, 37-48; (4) GwG-Info 1983, 52, 7-27; (5) Zeitschrift f. Personenzentr. Psychol. u. Psychotherapie 1984, 3, 277-300\"\n",
    "    - \"Kumulative Dissertation. _WEITERE ANGABEN_\" (12x)\n",
    "\n",
    "- \"Buchausgabe\" (1x)\n",
    "- \"Dissertation\" (1x, BE: SR!)\n",
    "\n",
    "Complex:\n",
    "- ca 200x eine URL - oft DOI-Typ, Stichproben: das sind externe Supplements\n",
    "- Gesprächsführung: \n",
    "    danach 1 Name: Vorname Nachname (als Contribution mit Rolle interviewer exportieren?)\n",
    "    danach 2 Namen: Vorname Nachname, Vorname Nachname (als 2 Contributions mit Rolle interviewer exportieren?)\n",
    "- als **Buchausgabe** - mit oder ohne Titel:\n",
    "    - \"Original als Buchausgabe erschienen: YYYY\"\n",
    "    - 'Original als Buchausgabe unter dem Titel \"Psychologie und Neurophysiotherapie Vojtas. Ein Gruppenvergleich zwischen frühbehandelten und bisher unauffälligen Vorschulkindern\" erschienen: 1982'\n",
    "    - Original als Buchausgabe _unter dem Titel \"Kinderpsychotherapien. Schulenbildung, Schulenstreit, Integration\"_ erschienen: 1984\n",
    "- als **Band** einer **Report-Reihe**:\n",
    "    - Original als Band einer Report-Reihe erschienen: 1990 (Bad Tönissteiner Blätter. Beiträge zur Suchtforschung und -therapie. Schriftenreihe der Fachklinik Bad Tönisstein, Band 2, Heft 1)\n",
    "    - Original als Band einer Report-Reihe erschienen: 2000. (Beiträge zur Arbeitsmarkt- und Berufsforschung, BeitrAB 235)\n",
    "    - Original als Band einer Report-Reihe erschienen: 2000. (Forschungsbericht, Nr. 2000-27)\n",
    "- als **Teil** einer **Report-Reihe**:\n",
    "    - Original als Teil einer Report-Reihe erschienen: 1977\n",
    "- als **Beitrag** in einem **Sammelwerk**:\n",
    "    - Original als Beitrag in einem Sammelwerk erschienen: Holzkamp, Klaus (Ed.) 1979. Forum Kritische Psychologie 5\n",
    "\n",
    "- Original als Zeitschriftenaufsatz erschienen: 2001. Should courts order PAS-children to visit/reside with the alienated parent? In: American Journal of Forensic Psychlogy, 19 (3), p. 61-106\n",
    "- Original als on-line Version erschienen auf der Homepage der Zeitschrift Supervision: www.fpi-publikationen.de/supervision\n",
    "\n",
    "- Anlage: (17x)\n",
    "- Anhang (6x) ...\n",
    "- Auszug aus: (158x)\n",
    "- \"Auszug aus \" (5x) zb: 'dem 4. Kapitel von \"Conceptual foundations of occupational therapy\"'\n",
    "- Auszüge aus einem Briefwechsel ... mit Andreas Wilhelm, Auckland/Neuseeland\n",
    "\n",
    "- Auszüge aus dem Original: (gefolgt von YYYY. Titel. Ort: Verlag) (2x)\n",
    "- Auswahl aus dem Original: (gefolgt von YYYY. Titel. Ort: Verlag)\n",
    "\n",
    "- Ausgabe in X Bänden/Heften/Ringordner (54x)\n",
    "\n",
    "- Abdruck aus: (2x)\n",
    "\n",
    "\n",
    "- Published in: (12x)\n",
    "- Also published (in:, under the title, in German language)\n",
    "    - in:\n",
    "        - Also published in: (5x)\n",
    "        - Already published in: (2x)\n",
    "        - Also published in German language in: (1x)\n",
    "        - Auch erschienen in: (7x)\n",
    "        - Außerdem erschienen in: (1x)\n",
    "    - as:\n",
    "        - Also published in German language under the title (1x)\n",
    "        - Also published as: (1x)\n",
    "        - Also published under the title (18x)\n",
    "\n",
    "    - Bereits:\n",
    "        - Bereits erschienen in: (646x)\n",
    "        - Bereits in anderer Fassung erschienen in:  \n",
    "        - Bereits in ausführlicherer Form erschienen in:\n",
    "        - Bereits ausführlicher erschienen in:\n",
    "        - Bereits einer kürzeren Fassung erschienen in:\n",
    "        - Bereits leicht gekürzt in englischer Sprache erschienen in: \n",
    "        - Bereits in englischer Sprache erschienen in: \n",
    "        - Bereits in französischer Sprache erschienen in:\n",
    "        - Bereits in holländischer Sprache erschienen in:\n",
    "        - Bereits in niederländischer Sprache erschienen in:\n",
    "        - Bereits in spanischer Sprache erschienen in:\n",
    "        - Bereits in deutscher Sprache erschienen in:\n",
    "\n",
    "    - Auch als Buchausgabe erschienen: \n",
    "    - Auch als Buchausgabe erschienen unter dem Titel: XXXX. YYYY. Ort: Verlag. ISBN\n",
    "    - Auch als Buchausgabe unter dem Titel XXX erschienen:\n",
    "    - Auch als Buchfassung unter dem Titel: \"XXX\" erschienen: YYYY. Ort: Verlag \n",
    "    - Als Buchausgabe unter dem Titel ...\n",
    "\n",
    "- a correct (ion) (is published, was published, to this article was published) (48x)\n",
    "- Aus dem Englischen übersetzt von (6x)\n",
    "- Aus den ersten drei Kapiteln bestehende Übersetzung des Originals: 1955. The psychology of personal constructs. New York: Norton</BN>\n",
    "- <BN>Aktualisierte Fassung des Originals: 1985. AIDS - vår framtid? Stockholm: Svenska Carnegie Institutet</BN>\n",
    "- <BN>Als Autorenname sind die Initialen R. S. angegeben</BN>\n",
    "- <BN>Als Erstveröffentlichung erschienen in: Der Gynäkologe, (1) 1986 unter dem Titel \"Die weibliche Sexualität aus psychoanalytischer Sicht\"</BN>\n",
    " \n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get Research Data from URLAI and DATAC:\n",
    "\n",
    "We'll just put any contents of subfields |u and |d into one processing field, check what it actually is, and then sort it accordingly. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "-- URLAI field 0:--\n",
      "genre:researchData.\n",
      "bf:usageAndAccessPolicy > bf:AccessPolicy\n",
      "> rdfs:label: restricted access@en. > rdf:value 'http://purl.org/coar/access_right/c_16ec'^^xsd:anyURI\n",
      "bf:identifiedBy > bf:Doi > rdf:value: 10.5160/psychdata.stuh96ko20.\n",
      "\n",
      "-- URLAI field 1:--\n",
      "genre:researchData.\n",
      "bf:usageAndAccessPolicy > bf:AccessPolicy\n",
      "> rdfs:label: restricted access@en. > rdf:value 'http://purl.org/coar/access_right/c_16ec'^^xsd:anyURI\n",
      "bf:identifiedBy > bf:Doi > rdf:value: 10.5160/psychdata.wfcn13ma18.\n",
      "\n",
      "-- URLAI field 2:--\n",
      "genre:researchData.\n",
      "bf:usageAndAccessPolicy > bf:AccessPolicy\n",
      "> rdfs:label: restricted access@en. > rdf:value 'http://purl.org/coar/access_right/c_16ec'^^xsd:anyURI\n",
      "bf:electronicLocator > Res > rdf:value: https://osf.io/hafsx_view_only^^anyURI.\n",
      "\n",
      "-- URLAI field 3:--\n",
      "genre:researchData.\n",
      "bf:usageAndAccessPolicy > bf:AccessPolicy\n",
      "> rdfs:label: restricted access@en. > rdf:value 'http://purl.org/coar/access_right/c_16ec'^^xsd:anyURI\n",
      "bf:note > bf:Note > rdfs:label: 12 23  56\n"
     ]
    }
   ],
   "source": [
    "from distutils.command import build\n",
    "import html \n",
    "import re\n",
    "\n",
    "\n",
    "\n",
    "def build_doi_identifier_node(doi):\n",
    "    print(f\"bf:identifiedBy > bf:Doi > rdf:value: {doi}.\")\n",
    "\n",
    "\n",
    "def build_electronic_locator_node(url):\n",
    "    print(f\"bf:electronicLocator > Res > rdf:value: {url}^^anyURI.\")\n",
    "\n",
    "def check_for_url_or_doi(string):\n",
    "    \"\"\"checks if the content of the string is a doi or url or something else.\n",
    "       Returns the a string and a string_type (doi, url, unknown). The given string \n",
    "       is sanitized, eg. missing http protocol is added for urls; dois are stripped\n",
    "       of web protocols and domain/subdomains like dx, doi.org).\"\"\"\n",
    "    # first, # replace spaces with underscores:\n",
    "    string = re.sub(' {2,}', ' ', string)\n",
    "    string = re.sub(\" \", \"_\", string)\n",
    "    doi_pattern = re.compile(r\"^(https?:)?(\\/\\/)?(dx\\.)?doi\\.org\\/?(.*)$\")\n",
    "    if doi_pattern.search(string):\n",
    "        # remove the matching part:\n",
    "        string = doi_pattern.search(string).group(4)\n",
    "        string_type = \"doi\"\n",
    "        # print(\"DOI: \" + doi)\n",
    "    elif string.startswith(\"10.\"):\n",
    "        # if the string starts with \"10.\" the whole thing is a DOI:\n",
    "        string_type = \"doi\"\n",
    "        # print(\"DOI: \" + doi)\n",
    "        # proceed to generate an identifier node for the doi:\n",
    "    else:\n",
    "        # doi = None\n",
    "        # check for validity of url using a regex:\n",
    "        url_pattern = re.compile(r\"[(http(s)?):\\/\\/(www\\.)?a-zA-Z0-9@:%._\\+~#=]{2,256}\\.[a-z]{2,6}\\b([-a-zA-Z0-9@:%_\\+.~#?&//=]*)\", re.IGNORECASE)\n",
    "        if url_pattern.search(string):\n",
    "            # if it's a nonstandard url starting with \"//\", add a \"http:\" protocol to the start:\n",
    "            if string.startswith(\"//\"):\n",
    "                string = \"http:\" + string\n",
    "            string_type = \"url\"\n",
    "            # print(\"URL: \" + datac_url)\n",
    "        else:\n",
    "            # url = None\n",
    "            string_type = \"unknown\"\n",
    "            # print(\"Das ist weder eine DOI noch eine URL: \" + string)\n",
    "    return string, string_type\n",
    "\n",
    "def get_subfield(subfield_full_string, subfield_name):\n",
    "        # split out the subfield:\n",
    "        subfield = subfield_full_string.split(f\"|{subfield_name} \")[1].split(\" |\")[0]\n",
    "         # strip out any double spaces and replace them with single spaces:\n",
    "        subfield = re.sub(' {2,}', ' ', subfield)\n",
    "        return subfield \n",
    "\n",
    "def get_datac(datac_list):\n",
    "    \"\"\"Gets research data from field DATAC. \n",
    "Note: We define all data from this field as type \"research data only, no code\", and \"open/unrestricted access\"\n",
    "Newer data from PSYNDEXER may be something else, but for first migration, we assume all data is research data only.\n",
    "\"\"\"\n",
    "    # for datac in record.findall(\"DATAC\"):\n",
    "    # go through the list of datac fields and get the doi, if there is one:\n",
    "    for count, data in enumerate(datac_list):\n",
    "        print(f\"\\n-- Datac field {count}:--\")\n",
    "        print(\"genre:researchData.\")\n",
    "        print(\"bf:usageAndAccessPolicy > bf:AccessPolicy\")\n",
    "        print(\"> rdfs:label: open access@en. > rdf:value 'http://purl.org/coar/access_right/c_abf2'^^xsd:anyURI\")\n",
    "        # first of all, get the text in that field, cleaning it from html entities in the process.\n",
    "        # datac_field = datac.text.strip()\n",
    "        datac_field = data.strip()\n",
    "        # grab subfields u and d as strings and check if they are a url or a doi:\n",
    "        for subfield_name in (\"u\", \"d\"):\n",
    "            try: \n",
    "                subfield = get_subfield(datac_field, subfield_name)\n",
    "            except:\n",
    "                subfield = None\n",
    "            else:\n",
    "                # if the string_type returned [1] is doi or url, treat them accordingly, using the returned string [0]\n",
    "                # as a doi or url:\n",
    "                # if it is a doi, run a function to generate a doi identifier node\n",
    "                if check_for_url_or_doi(subfield)[1] == \"doi\":\n",
    "                    build_doi_identifier_node(check_for_url_or_doi(subfield)[0])\n",
    "                elif check_for_url_or_doi(subfield)[1] == \"url\":\n",
    "                    build_electronic_locator_node(check_for_url_or_doi(subfield)[0])\n",
    "                # if the returned typ is something else \"unknown\", do nothing with it:\n",
    "                else:\n",
    "                    print(\"bf:note > bf:Note > rdfs:label: \" + subfield)\n",
    "                    \n",
    "datac = (\n",
    "    \"|u https://zenodo.org/record/160530 |d 10.5281/zenodo.160530\",\n",
    "    \"|u https://dx.doi.org/10.5281/zenodo.160530 |d 10.5281/zenodo.160530\",\n",
    "    \"|u http://dx.doi.org/10.1016/j.psyneuen.2015.11.018\",\n",
    "    \"|u 10.3389/fpsyg.2020.01623\",\n",
    "    \"|d 10.1016/j.jenvp.2020.101428\",\n",
    "    \"|u http://webapps.ccns.sbg.ac.at/OpenData |d \",\n",
    "    \"|u //osf.io/nj6zt/?view  only=b78ad5411b4b4e^Dffa15dc2c5fee17d6e\",\n",
    "    \"|u 123456789\",\n",
    "    \"|u https://static-content.springer.com/esm/art%3A10.1038%2Fs41598-018-25953-0/MediaObjects/41598 2018 25953 MOESM1 ESM.pdf\",\n",
    "\n",
    ")\n",
    "        \n",
    "def get_urlai(urlai_list):\n",
    "    \"\"\"Gets research data from field URLAI. This is always in PsychData, so it will be restricted access by default.\n",
    "    We will also assume it to always be just research data, not code.\n",
    "    \"\"\"\n",
    "    for count, data in enumerate(urlai_list):\n",
    "        print(f\"\\n-- URLAI field {count}:--\")\n",
    "        print(\"genre:researchData.\")\n",
    "        print(\"bf:usageAndAccessPolicy > bf:AccessPolicy\")\n",
    "        print(\"> rdfs:label: restricted access@en. > rdf:value 'http://purl.org/coar/access_right/c_16ec'^^xsd:anyURI\")\n",
    "        urlai_field = data.strip()\n",
    "        # there are no subfields in urlai, so let's just grab the whole thing and pass it on to the url or doi checker:\n",
    "        # if the string_type returned [1] is doi or url, treat them accordingly, using the returned string [0]\n",
    "        # as a doi or url:\n",
    "        # if it is a doi, run a function to generate a doi identifier node\n",
    "        if check_for_url_or_doi(urlai_field)[1] == \"doi\":\n",
    "            build_doi_identifier_node(check_for_url_or_doi(urlai_field)[0])\n",
    "        elif check_for_url_or_doi(urlai_field)[1] == \"url\":\n",
    "            build_electronic_locator_node(check_for_url_or_doi(urlai_field)[0])\n",
    "        # if the returned typ is something else \"unknown\", do nothing with it:\n",
    "        else:\n",
    "            print(\"bf:note > bf:Note > rdfs:label: \" + urlai_field)\n",
    "\n",
    "\n",
    "\n",
    "urlais = (\"http://dx.doi.org/10.5160/psychdata.stuh96ko20\", \n",
    "          \"https://doi.org/10.5160/psychdata.wfcn13ma18\", \n",
    "          \"https://osf.io/hafsx  view only\", \"12 23  56\"\n",
    ")\n",
    "\n",
    "#get_datac(datac)\n",
    "\n",
    "get_urlai(urlais)\n",
    "\n",
    "\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It's best if we had a generic function that builds a research data node.\n",
    "\n",
    "If passed a URLAI field, we build one with restricted access,\n",
    "if passed a DATAC field, we build one with open access.\n",
    "\n",
    "Otherwise, they should be treated the same! How can this be done? With a parameter?\n",
    "\n",
    "When calling the function (we call it twice, one for all URLAIs, once for all DATACs?)?\n",
    "\n",
    "Or we could call it once, and it will go through all URLAIs and DATACs of the record?\n",
    "\n",
    "What should be in the function, anyway:\n",
    "\n",
    "- build a bnode for the relationship, add relation, \n",
    "- build a bnode for the supplement work\n",
    "- build a bnode for the supplement instance\n",
    "- add the doi identifier node or electroniclocator, depending on url or doi, or a note if it's something else\n",
    "- add the usage and access policy, which will be different depending on the source (URLAI or DATAC)\n",
    "- URLAIs don't have subfields, but DATACs do. \n",
    "\n",
    "maybe we should make a function that generates relationships, and then reuse it for all kinds of fields?\n",
    "Depending on the field type, we can make some changes (the relation, the genre of the related work)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[a rdfg:Graph;rdflib:storage [a rdflib:Store;rdfs:label 'Memory']].\n"
     ]
    }
   ],
   "source": [
    "from rdflib import Graph, Literal\n",
    "from rdflib.namespace import RDF, RDFS, XSD, Namespace\n",
    "from rdflib import BNode\n",
    "from rdflib import URIRef\n",
    "\n",
    "BF = Namespace(\"http://id.loc.gov/ontologies/bibframe/\")\n",
    "BFLC = Namespace(\"http://id.loc.gov/ontologies/bflc/\")\n",
    "MADS = Namespace(\"http://www.loc.gov/mads/rdf/v1#\")\n",
    "SCHEMA = Namespace(\"https://schema.org/\")\n",
    "WORKS = Namespace(\"https://w3id.org/zpid/resources/works/\")\n",
    "INSTANCES = Namespace(\"https://w3id.org/zpid/resources/instances/\")\n",
    "PXC = Namespace(\"https://w3id.org/zpid/ontology/classes/\")\n",
    "PXP = Namespace(\"https://w3id.org/zpid/ontology/properties/\")\n",
    "LANG = Namespace (\"http://id.loc.gov/vocabulary/iso639-2/\")\n",
    "LOCID = Namespace(\"http://id.loc.gov/vocabulary/identifiers/\")\n",
    "ROLES = Namespace(\"https://w3id.org/zpid/vocabs/roles/\")\n",
    "RELATIONS = Namespace(\"https://w3id.org/zpid/vocabs/relations/\")\n",
    "\n",
    "records_bf = Graph()\n",
    "\n",
    "records_bf.bind(\"bf\", BF) \n",
    "records_bf.bind(\"bflc\", BFLC) \n",
    "records_bf.bind(\"works\", WORKS)  \n",
    "records_bf.bind(\"instances\", INSTANCES) \n",
    "records_bf.bind(\"pxc\", PXC) \n",
    "records_bf.bind(\"pxp\", PXP) \n",
    "records_bf.bind(\"lang\", LANG) \n",
    "records_bf.bind(\"schema\", SCHEMA) \n",
    "records_bf.bind(\"locid\", LOCID) \n",
    "records_bf.bind(\"mads\", MADS) \n",
    "records_bf.bind(\"roles\", ROLES) \n",
    "records_bf.bind(\"relations\", RELATIONS)\n",
    "\n",
    "relation_types = {\n",
    "    \"rd_open_access\": {\n",
    "        \"relation\": \"hasResearchData\",\n",
    "        \"relatedTo_subprop\": \"supplement\",\n",
    "        \"work_subclass\": \"Dataset\",\n",
    "        \"content_type\": \"dataset\",\n",
    "        \"genre\": \"researchData\",\n",
    "        \"access_policy_label\": \"open access\",\n",
    "        \"access_policy_value\": \"http://purl.org/coar/access_right/c_abf2\"\n",
    "    },\n",
    "    \"rd_restricted_access\": {\n",
    "        \"relation\": \"hasResearchData\",\n",
    "        \"relatedTo_subprop\": \"supplement\",\n",
    "        \"work_subclass\": \"Dataset\",\n",
    "        \"content_type\": \"dataset\",\n",
    "        \"genre\": \"researchData\",\n",
    "        \"access_policy_label\": \"restricted access\",\n",
    "        \"access_policy_value\": \"http://purl.org/coar/access_right/c_16ec\"\n",
    "    },\n",
    "}\n",
    "\n",
    "import re\n",
    "\n",
    "\n",
    "\n",
    "def build_doi_identifier_node(instance, doi):\n",
    "    # print(f\"bf:identifiedBy > bf:Doi > rdf:value: {doi}.\")\n",
    "    # make bnode for the identifier:\n",
    "    identifier_node = BNode()\n",
    "    # give it class bf:Doi:\n",
    "    records_bf.add((identifier_node, RDF.type, BF.Doi))\n",
    "    # give it the doi as a literal value:\n",
    "    records_bf.add((identifier_node, RDF.value, Literal(doi)))\n",
    "    # attach it to the instance with bf:identifiedBy:\n",
    "    records_bf.add((instance, BF.identifiedBy, identifier_node))\n",
    "\n",
    "\n",
    "def build_electronic_locator_node(instance, url):\n",
    "    locator_node = BNode()\n",
    "    # add it to the instance_node of relationship_node via bf:electronicLocator:\n",
    "    # no specific class!\n",
    "    # give it the url as a literal value:\n",
    "    records_bf.set((locator_node, RDF.value, Literal(url, datatype=XSD.anyURI)))\n",
    "    # attach it to the instance with bf:electronicLocator:\n",
    "    records_bf.set((instance, BF.electronicLocator, locator_node))\n",
    "\n",
    "def build_note_node(instance, note):\n",
    "    note_node = BNode()\n",
    "    records_bf.set((note_node, RDF.type, BF.Note))\n",
    "    records_bf.set((note_node, RDFS.label, Literal(note)))\n",
    "    records_bf.set((instance, BF.note, note_node))\n",
    "\n",
    "def check_for_url_or_doi(string):\n",
    "    \"\"\"checks if the content of the string is a doi or url or something else.\n",
    "       Returns the a string and a string_type (doi, url, unknown). The given string \n",
    "       is sanitized, eg. missing http protocol is added for urls; dois are stripped\n",
    "       of web protocols and domain/subdomains like dx, doi.org).\"\"\"\n",
    "    # first, # replace spaces with underscores:\n",
    "    string = re.sub(' {2,}', ' ', string)\n",
    "    string = re.sub(\" \", \"_\", string)\n",
    "    doi_pattern = re.compile(r\"^(https?:)?(\\/\\/)?(dx\\.)?doi\\.org\\/?(.*)$\")\n",
    "    if doi_pattern.search(string):\n",
    "        # remove the matching part:\n",
    "        string = doi_pattern.search(string).group(4)\n",
    "        string_type = \"doi\"\n",
    "        # print(\"DOI: \" + doi)\n",
    "    elif string.startswith(\"10.\"):\n",
    "        # if the string starts with \"10.\" the whole thing is a DOI:\n",
    "        string_type = \"doi\"\n",
    "        # print(\"DOI: \" + doi)\n",
    "        # proceed to generate an identifier node for the doi:\n",
    "    else:\n",
    "        # doi = None\n",
    "        # check for validity of url using a regex:\n",
    "        url_pattern = re.compile(r\"[(http(s)?):\\/\\/(www\\.)?a-zA-Z0-9@:%._\\+~#=]{2,256}\\.[a-z]{2,6}\\b([-a-zA-Z0-9@:%_\\+.~#?&//=]*)\", re.IGNORECASE)\n",
    "        if url_pattern.search(string):\n",
    "            # if it's a nonstandard url starting with \"//\", add a \"http:\" protocol to the start:\n",
    "            if string.startswith(\"//\"):\n",
    "                string = \"http:\" + string\n",
    "            string_type = \"url\"\n",
    "            # print(\"URL: \" + datac_url)\n",
    "        else:\n",
    "            # url = None\n",
    "            string_type = \"unknown\"\n",
    "            # print(\"Das ist weder eine DOI noch eine URL: \" + string)\n",
    "    return string, string_type\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "        \n",
    "def get_urlai(work_uri, urlai_list):\n",
    "    \"\"\"Gets research data from field URLAI. This is always in PsychData, so it will be restricted access by default.\n",
    "    We will also assume it to always be just research data, not code.\n",
    "    \"\"\"\n",
    "    for data in urlai_list:\n",
    "        # print(f\"\\n-- URLAI field {count}:--\")\n",
    "        # print(\"genre:researchData.\")\n",
    "        # print(\"bf:usageAndAccessPolicy > bf:AccessPolicy\")\n",
    "        # print(\"> rdfs:label: restricted access@en. > rdf:value 'http://purl.org/coar/access_right/c_16ec'^^xsd:anyURI\")\n",
    "        urlai_field = data.strip()\n",
    "        doi_set = set()\n",
    "        #build the relationship node:\n",
    "        relationship_node, instance = build_work_relationship_node(work_uri, relation_type=\"rd_restricted_access\") \n",
    "        # there are no subfields in urlai, so let's just grab the whole thing and pass it on to the url or doi checker:\n",
    "        # if the string_type returned [1] is doi or url, treat them accordingly, using the returned string [0]\n",
    "        # as a doi or url:\n",
    "        # if it is a doi, run a function to generate a doi identifier node\n",
    "        if check_for_url_or_doi(urlai_field)[1] == \"doi\":\n",
    "            # build_doi_identifier_node(instance,check_for_url_or_doi(urlai_field)[0])\n",
    "            doi_set.add(check_for_url_or_doi(urlai_field)[0])\n",
    "        elif check_for_url_or_doi(urlai_field)[1] == \"url\":\n",
    "            build_electronic_locator_node(instance, check_for_url_or_doi(urlai_field)[0])\n",
    "        # if the returned typ is something else \"unknown\", do nothing with it:\n",
    "        else:\n",
    "            # print(\"bf:note > bf:Note > rdfs:label: \" + urlai_field)\n",
    "            build_note_node(instance, check_for_url_or_doi(urlai_field)[0])\n",
    "\n",
    "        # loop through the set to build doi nodes, so we won't have duplicates:\n",
    "        for doi in doi_set:\n",
    "            build_doi_identifier_node(instance, doi)\n",
    "        # now attach the finished node for the relationship to the work:\n",
    "        records_bf.add((work_uri, BFLC.relationship, relationship_node))\n",
    "\n",
    "\n",
    "\n",
    "urlais = (\"http://dx.doi.org/10.5160/psychdata.stuh96ko20\", \n",
    "          \"https://doi.org/10.5160/psychdata.wfcn13ma18\", \n",
    "          \"https://osf.io/hafsx  view only\", \"12 23  56\"\n",
    ")\n",
    "\n",
    "def build_work_relationship_node(work_uri, relation_type):\n",
    "    # check the relation_type against the relation_types dict:\n",
    "    if relation_type in relation_types:\n",
    "        # if it is, get the values for the relation_type:\n",
    "        relation = relation_types[relation_type][\"relation\"]\n",
    "        relatedTo_subprop = relation_types[relation_type][\"relatedTo_subprop\"]\n",
    "        work_subclass = relation_types[relation_type][\"work_subclass\"]\n",
    "        content_type = relation_types[relation_type][\"content_type\"]\n",
    "        genre = relation_types[relation_type][\"genre\"]\n",
    "        access_policy_label = relation_types[relation_type][\"access_policy_label\"]\n",
    "        access_policy_value = relation_types[relation_type][\"access_policy_value\"]\n",
    "    # make a bnode for this relationship:\n",
    "    relationship_bnode = BNode()\n",
    "    # make it class bflc:Relationship:\n",
    "    records_bf.set((relationship_bnode, RDF.type, BFLC.Relationship))\n",
    "    # add a bflc:Relation (with a label and value) via bflc:relation to the relationship bnode \n",
    "    # (label and value could be given as a parameter):\n",
    "    # print(\"\\tbflc:relation [a bflc:Relation ; rdfs:label 'has research data', rdf:value 'relation:hasResearchData'^^xsd:anyURI] ;\")\n",
    "    # relation_bnode = BNode()\n",
    "    # records_bf.set((relation_bnode, RDF.type, BFLC.Relation))\n",
    "    # records_bf.add((relation_bnode, RDFS.label, Literal(\"has research data\")))\n",
    "    # records_bf.add((relation_bnode, RDF.value, Literal(RELATIONS.hasResearchData)))\n",
    "    records_bf.set((relationship_bnode, BFLC.relation, URIRef(RELATIONS[relation])))\n",
    "    # make a bnode for the work:\n",
    "    related_work_bnode = BNode()\n",
    "    records_bf.add((related_work_bnode, RDF.type, BF.Work))\n",
    "    records_bf.add((related_work_bnode, RDF.type, URIRef(BF[work_subclass])))\n",
    "    # give work a content type:\n",
    "    records_bf.add((related_work_bnode, BF.content, Literal(content_type)))\n",
    "    # and a genre:\n",
    "    records_bf.add((related_work_bnode, BF.genre, Literal(genre)))\n",
    "    # attach the work bnode to the relationship bnode with bf:relatedTo \n",
    "    # (or a subproperty as given as a parameter)):\n",
    "    # print(\"\\tbf:relatedTo [a bf:Work ;\")\n",
    "    records_bf.add((relationship_bnode, BF[relatedTo_subprop], related_work_bnode))\n",
    "    # make a bnode for the instance:\n",
    "    related_instance_bnode = BNode()\n",
    "    records_bf.set((related_instance_bnode, RDF.type, BF.Instance))\n",
    "    records_bf.add((related_instance_bnode, RDF.type, BF.Electronic))\n",
    "    # attach the instance to the work bnode via bf:hasInstance:\n",
    "    #print(\"\\t\\tbf:hasInstance [a bf:Instance ;\")\n",
    "    records_bf.add((related_work_bnode, BF.hasInstance, related_instance_bnode))\n",
    "    # add accesspolicy to instance:\n",
    "    access_policy_node = BNode()\n",
    "    records_bf.add((access_policy_node, RDF.type, BF.AccessPolicy))\n",
    "    records_bf.add((access_policy_node, RDFS.label, Literal(access_policy_label, lang=\"en\")))\n",
    "    records_bf.add((access_policy_node, RDF.value, Literal(access_policy_value, datatype=XSD.anyURI)))\n",
    "    records_bf.add((related_instance_bnode, BF.usageAndAccessPolicy, access_policy_node))\n",
    "    # insert dois and/or urls:\n",
    "    #\n",
    "    # print(\"\\t\\t\\t]\") # end instance\n",
    "    # in the end, return the relationship bnode so it can be attached to the work\n",
    "    # records_bf.add((work_uri, BFLC.relationship, relationship_bnode))\n",
    "    return relationship_bnode, related_instance_bnode\n",
    "\n",
    "def get_datac(work_uri, datac_list):\n",
    "    \"\"\"Gets research data from field DATAC. \n",
    "Note: We define all data from this field as type \"research data only, no code\", and \"open/unrestricted access\"\n",
    "Newer data from PSYNDEXER may be something else, but for first migration, we assume all data is research data only.\n",
    "\"\"\"\n",
    "    # for datac in record.findall(\"DATAC\"):\n",
    "    # go through the list of datac fields and get the doi, if there is one:\n",
    "    for data in datac_list:\n",
    "        datac_field = data.strip()\n",
    "        # add an item \"hello\" to the set:\n",
    "        #build the relationship node:\n",
    "        relationship_node, instance = build_work_relationship_node(work_uri, relation_type=\"rd_open_access\") \n",
    "\n",
    "        # we want to drop any duplicate dois that can occur if datac has a doi and doi url (same doi, but protocol etc prefixed) \n",
    "        # for the same data that,\n",
    "        # after conversion, ends up being identical. So we make a set of dois,\n",
    "        # which we will add dois to, and then later loop through the set (sets are by defintion list with only unique items!):\n",
    "        doi_set = set()\n",
    "        # grab subfields u and d as strings and check if they are a url or a doi:\n",
    "        for subfield_name in (\"u\", \"d\"):\n",
    "            try: \n",
    "                subfield = get_subfield(datac_field, subfield_name)\n",
    "            except:\n",
    "                subfield = None\n",
    "            else:\n",
    "                # if the string_type returned [1] is doi or url, treat them accordingly, using the returned string [0]\n",
    "                # as a doi or url:\n",
    "                # if it is a doi, run a function to generate a doi identifier node\n",
    "                if check_for_url_or_doi(subfield)[1] == \"doi\":\n",
    "                    # add the doi to a list:\n",
    "                    doi_set.add(check_for_url_or_doi(subfield)[0])\n",
    "                    #build_doi_identifier_node(instance, check_for_url_or_doi(subfield)[0])\n",
    "                elif check_for_url_or_doi(subfield)[1] == \"url\":\n",
    "                    build_electronic_locator_node(instance, check_for_url_or_doi(subfield)[0])\n",
    "                    # if the returned typ is something else \"unknown\", do nothing with it:\n",
    "                else:\n",
    "                    # print(\"bf:note > bf:Note > rdfs:label: \" + subfield)\n",
    "                    build_note_node(instance, check_for_url_or_doi(subfield)[0])\n",
    "        # doi_set = set(doi_list)\n",
    "        # print(doi_list)\n",
    "        # print(doi_set)\n",
    "        for doi in doi_set:\n",
    "            build_doi_identifier_node(instance, doi)\n",
    "                \n",
    "                \n",
    "        # now attach the finished node for the relationship to the work:\n",
    "        records_bf.add((work_uri, BFLC.relationship, relationship_node))\n",
    "                    \n",
    "datac = (\n",
    "    \"|u https://zenodo.org/record/160530 |d 10.5281/zenodo.160530\",\n",
    "    \"|u https://dx.doi.org/10.5281/zenodo.160531 |d 10.5281/zenodo.160531\",\n",
    "    \"|u http://dx.doi.org/10.1016/j.psyneuen.2015.11.018\",\n",
    "    \"|u 10.3389/fpsyg.2020.01623\",\n",
    "    \"|d 10.1016/j.jenvp.2020.101428\",\n",
    "    \"|u http://webapps.ccns.sbg.ac.at/OpenData |d \",\n",
    "    \"|u //osf.io/nj6zt/?view  only=b78ad5411b4b4e^Dffa15dc2c5fee17d6e\",\n",
    "    \"|u 123456789\",\n",
    "    \"|u https://static-content.springer.com/esm/art%3A10.1038%2Fs41598-018-25953-0/MediaObjects/41598 2018 25953 MOESM1 ESM.pdf\",\n",
    "\n",
    ")\n",
    "\n",
    "## now for the main program:\n",
    "# make a Work node:\n",
    "work_uri = URIRef(WORKS[\"123456789\"])\n",
    "records_bf.add((work_uri, RDF.type, BF.Work))\n",
    "# call the function to build a relationship node, which should attach the relationship node to the work node:\n",
    "# build_work_relationship_node(work_uri, relation_type=\"rd_open_access\") \n",
    "\n",
    "# it's better to call the build_relationshjp_node function from within the urlai and datac functions. i supppose.\n",
    "# get_urlai(work_uri, urlais)\n",
    "get_datac(work_uri, datac)\n",
    "\n",
    "# serialize the graph:\n",
    "print(records_bf.serialize(\"dois.ttl\",format=\"turtle\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "yes, https://osf.io/atc48/ contains ATC48\n"
     ]
    }
   ],
   "source": [
    "url = \"https://osf.io/atc48/\"\n",
    "doi = \"10.17605/OSF.IO/ATC48\"\n",
    "#doi = \"10.17605/ui/ATC48\"\n",
    "if \"osf.io\" in url and \"OSF.IO/\" in doi and doi.split(\"/\")[2].lower() in url:\n",
    "    print(f\"duplicate doi in url {url}: {doi.split('/')[2]} from {doi}. Removing url in favor of doi.\")\n",
    "else:\n",
    "    print(\"no\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0000-0002-818X-844X is not a valid orcid.\n",
      "after corrections, 0000-0001-6000-0967 is valid.\n",
      "after corrections, 0000-0002-0089-7618 is valid.\n",
      "after corrections, 0000-0003-4757-1460 is valid.\n",
      "after corrections, 0000-0001-8112-0837 is valid.\n",
      "Velten,Julia is not a valid orcid.\n",
      "10.1007/978-3-658-10947-921-1 is not a valid orcid.\n",
      "10.1026/0033-3042/a000591 is not a valid orcid.\n",
      "10.1026/0049-8637/a000177 is not a valid orcid.\n",
      "after corrections, 0000-0002-8181-844X is valid.\n",
      "after corrections, 0000-0002-1397-0060 is valid.\n",
      "after corrections, 0000-0003-1342-7006 is valid.\n",
      "after corrections, 0000-0001-9885-3252 is valid.\n"
     ]
    }
   ],
   "source": [
    "from operator import ne\n",
    "import re\n",
    "\n",
    "def orcid_checker(orcid):\n",
    "    \"\"\"Checks if an orcid is valid. Returns True if valid, False if not.\"\"\"\n",
    "    # first, check if other stuff is at the beginning of the string - if it starts with either \"/\" or \"https://orcid.org/\" or \"orcid.org/\" - then strip that out, using regex:\n",
    "    # orcid = re.sub(r\"^(\\/|https?:\\/\\/(orcid\\.)?org\\/)?\", \"\", orcid)\n",
    "\n",
    "    # then remove any spaces:\n",
    "    orcid = orcid.replace(\" \", \"\")\n",
    "\n",
    "    # then check if it is a valid orcid by using a regex, which also checks if it starts with \"http(s)://orcid.org/\", orcid/org/\", or a \"/\" and removes these\":\n",
    "    orcid_pattern = re.compile(r\"^(https?:\\/\\/(orcid\\.)?org\\/)?(orcid\\.org\\/)?(\\/)?([0-9]{4}-[0-9]{4}-[0-9]{4}-[0-9]{3}[0-9X])$\")\n",
    "    if orcid_pattern.search(orcid):\n",
    "        # if it is, remove the matching part:\n",
    "        orcid = orcid_pattern.search(orcid).group(5)\n",
    "        print(f\"after corrections, {orcid} is valid.\")\n",
    "    else:\n",
    "        print(f\"{orcid} is not a valid orcid.\")\n",
    "\n",
    "        \n",
    "\n",
    "orcid_list = (\n",
    "    \"0000-0002-818X-844X\", # incorrect, because X in third group\n",
    "    \"0000-0001-6000- 0967\", # a correct orcid, but with a space inside: catch and correct, maybe look if it exists\n",
    "    \"0000-0002- 0089-7618\", # as above\n",
    "    \"/0000-0003-4757-1460\", # as above, but with a slash (c&p error)\n",
    "    \"/0000-0001-8112-0837\", # as above\n",
    "    \"Velten, Julia\", # unacceptable, drop\n",
    "    \"10.1007/978-3-658-10947-9 21-1\", # unacceptable, drop\n",
    "    \"10.1026/0033-3042/a000591\",\n",
    "    \"10.1026/0049-8637/a000177\",\n",
    "    \"orcid.org/0000-0002-8181-844X\", # remove orcid.org/ and check if valid orcid\n",
    "    \"https://orcid.org/0000-0002-1397-0060\", # remove https://orcid.org/ and check if valid orcid\n",
    "    \"https://orcid.org/0000-0003-1342-7006\",\n",
    "    \"https://orcid.org/0000-00 01-9885- 3252\",\n",
    ")\n",
    "\n",
    "for orcid in orcid_list:\n",
    "    orcid_checker(orcid)\n",
    "\n",
    "\n",
    "orcid_field = (\n",
    "    \"0000-0003-3359-6157 |u Zinke, Alexander\", # switch name and orcid\n",
    "    \"0000-0002-0350-1359 |u Soloviev, Andrey G.\",\n",
    "    \"0000-0001-8311-1184 |u Peseschkian, Hamid\"\n",
    "    \"Knaevelsrud, Christine |u Knaevelsrud, Christine\", # drop orcid\n",
    "    \"Heinzel, Carlotta V. |u 0000-0002-2619-913X\" # correct, leave as is\n",
    ")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tp: 0, fp: 1, mod: 1 - prec: 0.5\n",
      "tp: 1, fp: 1, mod: 1 - prec: 1.3333333333333333\n",
      "tp: 0, fp: 18, mod: 1 - prec: 0.05263157894736842\n",
      "tp: 1, fp: 18, mod: 1 - prec: 1.05\n"
     ]
    }
   ],
   "source": [
    "values = ({\"tp\": 0, \"fp\": 1},\n",
    "          {\"tp\": 1, \"fp\": 1},\n",
    "            {\"tp\": 0, \"fp\": 18},\n",
    "            {\"tp\": 1, \"fp\": 18},\n",
    "          )\n",
    "\n",
    "modifier = 1\n",
    "\n",
    "def modified_precision(tp, fp, modifier):\n",
    "    \"\"\"Calculates precision from true positives and false positives.\"\"\"\n",
    "    return (tp+modifier / (tp + modifier + fp))\n",
    "\n",
    "for value in values:\n",
    "    print(f'tp: {value[\"tp\"]}, fp: {value[\"fp\"]}, mod: {modifier} - prec: {modified_precision(value[\"tp\"], value[\"fp\"], modifier)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Getting and modelling links to replicated studies in field RPLIC\n",
    "\n",
    "Es gibt keine Subfelder. Im Hauptfeld steht meist ein Zitationsstring. Er kann manchmal auch eine DOI enthalten, entwerder ausschlielßich oder am Ende der Zitation (oder irgendwo dazwischen).\n",
    "Oder auch eine URL, zB zu OSF.\n",
    "\n",
    "Am einfachsten könnte sein, die Zitation an Crossref weiterzugeben und sich eine DOI dafür zurückliefern zu lassen.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1: Testeintrag, wieder loeschen: \n",
      "https://doi.org/10.1002/chin.197104180\n",
      "2: Title mismatch: OSF io  - Materi Jeff Webinar EIYRA 2 != https://osf.io/kv65n/: \n",
      "https://doi.org/None\n",
      "3: Timmer, K., Calabria, M., Branzi, F. M., Baus, C., & Costa, A. (2018). On the reliability of switching costs across time and domains. Frontiers in Psychology, 9, 1032.: \n",
      "https://doi.org/10.3389/fpsyg.2018.01032\n",
      "4: Title mismatch: Dream characters and the dream ego: An exploratory online study in lucid dreams. != Stumbrys, T., & Erlacher, D. (2014). The science of lucid dream induction. In R. Hurd, & K. Bulkeley (Eds.). Lucid dreaming: New perspectives on consciousness in sleep (pp. 77–96). Westport: Praeger.: \n",
      "https://doi.org/None\n",
      "5: Mattler, U., & Fendrich, R. (2010). Consciousness mediated by neural transition states: How invisiblyrapid motions can become visible.Consciousness and Cognition,19, 172–185. https://doi.org/10.1016/j.concog.2009.12.015: \n",
      "https://doi.org/10.1016/j.concog.2009.12.015\n",
      "6: Dreßing H, Kuehner C, Gass P: Lifetime prevalence and impact of stalking in a European population: epidemiological data from a middlesized German city. Br J Psychiat 2005; 187: 168–72: \n",
      "https://doi.org/10.1192/bjp.187.2.168\n",
      "7: Daw ND, Oğt;'Doherty JP, Dayan P, Seymour B, Dolan RJ. 2006. Cortical substrates for exploratory decisions inhumans.Nature441:876–879.DOI: https://doi.org/10.1038/nature04766,PMID: 16778890: \n",
      "https://doi.org/10.1038/nature04766\n",
      "8: Kahneman, D., & Tversky. A. (1979). Prospect theory: An analysis of decision under risk. Econometrica, 47(2), 263-292.: \n",
      "https://doi.org/10.2307/1914185\n",
      "9: Saalbach, H., Eckstein, D., Andri, N., Hobi, R., & Grabner, R. H. (2013). When language of instruction and language of application differ: Cognitive costs of bilingual mathematics learning. Learning and Instruction, 26, 36-44.: \n",
      "https://doi.org/10.1016/j.learninstruc.2013.01.002\n",
      "10: Grabner, R. H., Saalbach, H., & Eckstein, D. (2012). Language‐Switching Costs in Bilingual Mathematics Learning. Mind, Brain, and Education, 6(3), 147-155.: \n",
      "https://doi.org/10.1111/j.1751-228x.2012.01150.x\n",
      "11: Eskine, K. J., Kacinik, N. A., & Prinz, J. J. (2011). A bad taste in the mouth: Gustatory disgustinfluences moral judgment.Psychological Science,22(3), 295–299.: \n",
      "https://doi.org/10.1177/0956797611398497\n",
      "12: Head, D., & Isom, M. (2010). Age effects on wayfinding and route learn-ing skills.Behavioural Brain Research,209,49–58.: \n",
      "https://doi.org/10.1016/j.bbr.2010.01.012\n",
      "13: Rütsche, B., Hauser, T. U., Jäncke, L., and Grabner, R. H. (2015). Whenproblem size matters: differential effects of brain stimulation on arithmeticproblem solving and neural oscillations. PLoS ONE10:e0120665.doi: 10.1371/journal.pone.0120665: \n",
      "https://doi.org/10.1371/journal.pone.0120665\n",
      "14: Title mismatch: Front cover != https://doi.org/10.1101/470435: \n",
      "https://doi.org/None\n",
      "15: Armstrong, T., Bilsky, S. A., Zhao, M., & Olatunji, B. O. (2013). Dwellingon potential threat cues: An eye movement marker for combat-relatedPTSD.Depression and Anxiety, 30,497–502.: \n",
      "https://doi.org/10.1002/da.22115\n",
      "16: Kugler, K. G., Reif, J. A. M., Kaschner, T., & Brodbeck, F. C. (2018). Gender differences in the initiation of negoti-ations: A meta-analysis.Psychological Bulletin,144, 198–222.: \n",
      "https://doi.org/10.1037/bul0000135\n",
      "17: Tietze, W., Becker-Stoll, F., Bensel, J., Eckhardt, A. G., Haug-Schnabel, G., Kalicki, B.,. . . & Leyendecker, B. (2013). Nationale Untersuchung zur Bildung, Betreuung undErziehung in der frühen Kindheit (NUBBEK). Weimar: verlag das netz.: \n",
      "https://doi.org/10.3224/zff.v26i2.16528\n",
      "18: Korb, F. M., Jiang, J., King, J. A., & Egner, T. (2017). Hierarchicallyorganized medial frontal cortex-basal ganglia loops selectively controltask- and response-selection.Journal of Neuroscience: The Official Journal of the Society for Neuroscience, 37,7893–7905.http://dx.doi.org/10.1523/JNEUROSCI.3289-16.2017: \n",
      "https://doi.org/10.1523/jneurosci.3289-16.2017\n",
      "19: Langguth, B. et al. Tinnitus severity, depression, and the big five personality traits. In Langguth, B., Hajak, G., Kleinjung, T., Cacace, A. & Møller, A. R. (eds.) Progress in Brain Research, vol. 166 of Tinnitus: Pathophysiology and Treatment, 221–225.: \n",
      "https://doi.org/10.1016/s0079-6123(07)66020-8\n",
      "20: Schmitz, J., Krämer, M., Blechert, J., & Tuschen-Caffier, B. (2010). Post-event processing in children with social phobia.Journal ofAbnormal Child Psychology, 38(7), 911–919.: \n",
      "https://doi.org/10.1007/s10802-010-9421-2\n",
      "21: Gamble, T., & Walker, I. (2016). Wearing a bicycle helmet can increase risk taking and sensation seeking in adults. Psychological Science, 27(2), 289–294. https ://doi.org/10.1177/09567 97615 620784: \n",
      "https://doi.org/10.1177/0956797615620784\n",
      "22: Sprenger, L., Becker, K., Heinzel-Gutenbrunner, M., Mingebach, T., Otterbach, S., Peters, M., & Kamp-Becker, I. (2015). Is the ^D&lt;,Stepping Stones/Triple Pğt;\"-parenting program a reason-able, additional intervention in the treatment of children with an autism spectrum disorder? Kindheit und Entwicklung, 24, 28–36: \n",
      "https://doi.org/10.1026/0942-5403/a000156\n",
      "23: Hagan CC, Graham JM, Tait R, et al. Adolescents with current major depressive disorder show dissimilar patterns of age-related differences in ACC and thalamus. Neuroimage Clin 2015;7:391-9.: \n",
      "https://doi.org/10.1016/j.nicl.2014.12.019\n",
      "24: Jaworska N, Yucel K, Courtright A, et al. Subgenual anterior cin-gulate cortex and hippocampal volumes in depressed youth: the role of comorbidity and age. J Affect Disord 2016;190:726-32.: \n",
      "https://doi.org/10.1016/j.jad.2015.10.064\n",
      "25: Tran, U. S., Glück, T. M., & Nader, I. W. (2013). Investigating the FiveFacet Mindfulness Questionnaire (FFMQ): construction of a shortform and evidence of a two-factor higher order structure of mind-fulness.Journal of Clinical Psychology, 69,951–965.https://doi.org/10.1002/jclp.21996.: \n",
      "https://doi.org/10.1002/jclp.21996\n",
      "26: Rand, D. G., Greene, J. D., & Nowak, M. A. (2012). Spontaneous giving and calculated greed. Nature, 489, 427–430: \n",
      "https://doi.org/10.1038/nature11467\n"
     ]
    }
   ],
   "source": [
    "# Looking up a citation string from RPLIC and getting the corresponding DOI from crossref:\n",
    "import re\n",
    "import requests\n",
    "import requests_cache\n",
    "import json\n",
    "from datetime import timedelta\n",
    "import modules.mappings as mappings\n",
    "from fuzzywuzzy import fuzz\n",
    "\n",
    "\n",
    "CROSSREF_FRIENDLY_MAIL = \"&mailto=ttr@leibniz-psychology.org\"\n",
    "# for getting a list of funders from api ():\n",
    "CROSSREF_API_URL = \"https://api.crossref.org/works?query=\"\n",
    "    \n",
    "urls_expire_after = {\n",
    "    # Custom cache duration per url, 0 means \"don't cache\"\n",
    "    # f'{SKOSMOS_URL}/rest/v1/label?uri=https%3A//w3id.org/zpid/vocabs/terms/09183&lang=de': 0,\n",
    "    # f'{SKOSMOS_URL}/rest/v1/label?uri=https%3A//w3id.org/zpid/vocabs/terms/': 0,\n",
    "}\n",
    "\n",
    "session = requests_cache.CachedSession(\n",
    "    \".cache/requests\",\n",
    "    allowable_codes=[200, 404],\n",
    "    expire_after=timedelta(days=30),\n",
    "    urls_expire_after=urls_expire_after,\n",
    ")\n",
    "\n",
    "def get_subfield(subfield_full_string, subfield_name):\n",
    "    \"\"\"Given a string that contains star subfields (|name ) and the name of the subfield,\n",
    "    e.g. i for |i, return the content of only that subfield as a string.\"\"\"\n",
    "    # first, make sure that the extracted substring is not None, not empty or completely comprised of spaces:\n",
    "    if subfield_full_string is not None and subfield_full_string != \"\":\n",
    "        # strip out any double spaces and replace with single space, also strip spaces around:\n",
    "        subfield_full_string = re.sub(\" {2,}\", \" \", subfield_full_string.strip())\n",
    "        # split out the content of the field - from the first |name to either the next | or the end of subfield_full_string:\n",
    "        subfield = None\n",
    "        # check if the subfield is in the string:\n",
    "        if f\"|{subfield_name}\" in subfield_full_string:\n",
    "            # if it is, split the string on the subfield name:\n",
    "            subfield = subfield_full_string.split(f\"|{subfield_name}\")[1].strip()\n",
    "            # end the string at the next | or the end of the string:\n",
    "            subfield = subfield.split(\"|\")[0].strip()\n",
    "        # subfield = subfield_full_string.split(f\"|{subfield_name}\")[1].strip().split(\"|\")[0].strip()\n",
    "        # print(subfield)\n",
    "        if subfield != \"\" and subfield is not None:\n",
    "            return html.unescape(mappings.replace_encodings(subfield))\n",
    "        else:\n",
    "            return None\n",
    "\n",
    "\n",
    "def get_mainfield(field_fullstring):\n",
    "    \"\"\"Given a string extracted from a star field that may have substrings or not, return the content of\n",
    "    the main field as a string - either to the first |subfield or the end of the field, if no subfields.\n",
    "    \"\"\"\n",
    "    # first, make sure that the extracted substring is not None, not empty or completely comprised of spaces:\n",
    "    if field_fullstring is not None and field_fullstring != \"\":\n",
    "        # strip out any double spaces and replace with single space, also strip spaces around:\n",
    "        field_fullstring = re.sub(\" {2,}\", \" \", field_fullstring.strip())\n",
    "        # split out the content of the field - to the first | or the end of subfield_full_string:\n",
    "        field = None\n",
    "        # check if a subfield is in the string:\n",
    "        if f\"|\" in field_fullstring:\n",
    "            # if it is, return the part before it:\n",
    "            field = field_fullstring.split(\"|\")[0].strip()\n",
    "        else:\n",
    "            # if not, return the whole string:\n",
    "            field = field_fullstring.strip()\n",
    "        if field != \"\" and field is not None:\n",
    "            return html.unescape(mappings.replace_encodings(field))\n",
    "        else:\n",
    "            return None\n",
    "\n",
    "rplics = (\n",
    "    \"Testeintrag, wieder loeschen\",\n",
    "    \"https://osf.io/kv65n/\",\n",
    "    \"Timmer, K., Calabria, M., Branzi, F. M., Baus, C., &amp; Costa, A. (2018). On the reliability of switching costs  across time and domains. Frontiers in Psychology, 9, 1032.\",\n",
    "    \"Stumbrys, T., &amp; Erlacher, D. (2014). The science of lucid dream induction. In R. Hurd, &amp; K. Bulkeley (Eds.). Lucid dreaming: New perspectives on consciousness in sleep (pp. 77^DDS96). Westport: Praeger.\",\n",
    "    \"Mattler, U., &amp; Fendrich, R. (2010). Consciousness mediated by neural transition states: How invisiblyrapid motions can become visible.Consciousness and Cognition,19, 172^DDS185. https://doi.org/10.1016/j.concog.2009.12.015\",\n",
    "    \"Dreßing H, Kuehner C, Gass P: Lifetime prevalence and impact of stalking in a European population: epidemiological data from a middlesized German city. Br J Psychiat 2005; 187: 168^DDS72\",\n",
    "    \"Daw ND, O^D&gt;'Doherty JP, Dayan P, Seymour B, Dolan RJ. 2006. Cortical substrates for exploratory decisions inhumans.Nature441:876^DDS879.DOI: https://doi.org/10.1038/nature04766,PMID: 16778890\",\n",
    "    \"Kahneman, D., &amp; Tversky. A. (1979). Prospect theory: An analysis of decision under risk. Econometrica, 47(2), 263-292.\",\n",
    "    \"Saalbach, H., Eckstein, D., Andri, N., Hobi, R., &amp; Grabner, R. H. (2013). When language of instruction and language of application differ: Cognitive costs of bilingual mathematics learning. Learning and Instruction, 26, 36-44.\",\n",
    "    \"Grabner, R. H., Saalbach, H., &amp; Eckstein, D. (2012). Language‐Switching Costs in Bilingual Mathematics Learning. Mind, Brain, and Education, 6(3), 147-155.\",\n",
    "    \"Eskine, K. J., Kacinik, N. A., &amp; Prinz, J. J. (2011). A bad taste in the mouth: Gustatory disgustinfluences moral judgment.Psychological Science,22(3), 295^DDS299.\",\n",
    "    \"Head, D., &amp; Isom, M. (2010). Age effects on wayfinding and route learn-ing skills.Behavioural Brain Research,209,49^DDS58.\",\n",
    "    \"Rütsche, B., Hauser, T. U., Jäncke, L., and Grabner, R. H. (2015). Whenproblem size matters: differential effects of brain stimulation on arithmeticproblem  solving  and  neural  oscillations. PLoS ONE10:e0120665.doi: 10.1371/journal.pone.0120665\",\n",
    "    \"https://doi.org/10.1101/470435\",\n",
    "    \"Armstrong, T., Bilsky, S. A., Zhao, M., &amp; Olatunji, B. O. (2013). Dwellingon potential threat cues: An eye movement marker for combat-relatedPTSD.Depression and Anxiety, 30,497^DDS502.\",\n",
    "    \"Kugler, K. G., Reif, J. A. M., Kaschner, T., &amp; Brodbeck, F. C. (2018). Gender differences in the initiation of negoti-ations: A meta-analysis.Psychological Bulletin,144, 198^DDS222.\",\n",
    "    \"Tietze, W., Becker-Stoll, F., Bensel, J., Eckhardt, A. G., Haug-Schnabel, G., Kalicki, B.,. . . &amp; Leyendecker, B. (2013). Nationale Untersuchung zur Bildung, Betreuung undErziehung in der frühen Kindheit (NUBBEK). Weimar: verlag das netz.\",\n",
    "    \"Korb, F. M., Jiang, J., King, J. A., &amp; Egner, T. (2017). Hierarchicallyorganized medial frontal cortex-basal ganglia loops selectively controltask- and response-selection.Journal of Neuroscience: The Official Journal of the Society for Neuroscience, 37,7893^DDS7905.http://dx.doi.org/10.1523/JNEUROSCI.3289-16.2017\",\n",
    "    \"Langguth,  B. et al. Tinnitus severity, depression, and the big five personality traits. In Langguth, B., Hajak, G., Kleinjung, T., Cacace, A. &amp; Møller, A. R. (eds.) Progress in Brain Research, vol. 166 of Tinnitus: Pathophysiology and Treatment, 221^DDS225.\",\n",
    "    \"Schmitz, J., Krämer, M., Blechert, J., &amp; Tuschen-Caffier, B. (2010). Post-event processing in children with social phobia.Journal ofAbnormal Child Psychology, 38(7), 911^DDS919.\",\n",
    "    \"Gamble, T., &amp; Walker, I. (2016). Wearing a bicycle helmet can increase risk taking and sensation seeking in adults. Psychological Science, 27(2),  289^DDS294.  https ://doi.org/10.1177/09567 97615 620784\",\n",
    "    'Sprenger,  L.,  Becker,  K.,  Heinzel-Gutenbrunner,  M.,  Mingebach,  T.,  Otterbach,  S.,  Peters,  M.,  &amp;  Kamp-Becker,  I.  (2015).  Is  the  ^D&lt;,Stepping  Stones/Triple  P^D&gt;\"-parenting  program  a  reason-able, additional intervention in the treatment of children with an autism spectrum disorder? Kindheit und Entwicklung, 24, 28^DDS36',\n",
    "    \"Hagan  CC,  Graham  JM,  Tait  R,  et  al.  Adolescents  with  current  major  depressive  disorder  show  dissimilar  patterns  of  age-related  differences in ACC and thalamus. Neuroimage Clin 2015;7:391-9.\",\n",
    "    \"Jaworska  N,  Yucel  K,  Courtright  A,  et  al.  Subgenual  anterior  cin-gulate  cortex  and  hippocampal  volumes  in  depressed  youth:  the  role of comorbidity and age. J Affect Disord 2016;190:726-32.\",\n",
    "    \"Tran, U. S., Glück, T. M., &amp; Nader, I. W. (2013). Investigating the FiveFacet Mindfulness Questionnaire (FFMQ): construction of a shortform and evidence of a two-factor higher order structure of mind-fulness.Journal of Clinical Psychology, 69,951^DDS965.https://doi.org/10.1002/jclp.21996.\",\n",
    "    \" Rand, D. G., Greene, J. D., &amp; Nowak, M. A. (2012). Spontaneous giving and calculated greed. Nature, 489, 427^DDS430\",\n",
    "    )\n",
    "\n",
    "def build_replicated_study_relationship_node(rplic):\n",
    "        # extract subfields:\n",
    "    try:\n",
    "        replicated_doi = get_subfield(rplic, \"d\")\n",
    "    except:\n",
    "        replicated_doi = None\n",
    "    try:\n",
    "        replicated_url = get_subfield(rplic, \"u\")\n",
    "    except:\n",
    "        replicated_url = None\n",
    "    try:\n",
    "        replicated_citation = get_mainfield(rplic)\n",
    "    except:\n",
    "        replicated_citation = None\n",
    "    # if there is a url, add it as bf:electronicLocator:\n",
    "    if replicated_url:\n",
    "        print(f\"URL: {replicated_url}\")\n",
    "    # if there is a doi, add it as bf:identifier:\n",
    "    if replicated_doi:\n",
    "        print(f\"DOI: {replicated_doi}\")\n",
    "    # if there is a citation, look it up in crossref:\n",
    "    if replicated_citation:\n",
    "        print(f\"Citation: {replicated_citation}\")\n",
    "        fetch_doi_from_citation_string(replicated_citation)\n",
    "        print(f\"DOI: {doi}\")\n",
    "        print(f\"URL: https://doi.org/{doi}\")\n",
    "        \n",
    "        \n",
    "\n",
    "def fetch_doi_from_citation_string(citation):\n",
    "    \"\"\"Given a citation string, look up the DOI in crossref. Returns the citation string and the DOI.\"\"\"\n",
    "    # clean up ^DD codes with mappings.replace_encodings:\n",
    "    citation = mappings.replace_encodings(citation)\n",
    "    # the citation must not include any &amp or & - because they would  be interpreted params in the url:\n",
    "    citation = citation.replace(\"&amp;\", \"&\")\n",
    "    # citation = citation.replace(\"&\", \"and\")\n",
    "    \n",
    "    # strip out any double spaces and replace them with single spaces:\n",
    "    citation = re.sub(' {2,}', ' ', citation)\n",
    "    # strip out any spaces at the beginning or end:\n",
    "    clean_citation = citation.strip()\n",
    "    # query api, but put citation string in quotes:\n",
    "    # citation = f'\"{citation}\"'\n",
    "    #encode the string so it works as a url query:\n",
    "    citation = requests.utils.quote(clean_citation)\n",
    "    crossref_api_url = CROSSREF_API_URL + citation + CROSSREF_FRIENDLY_MAIL\n",
    "\n",
    "    # make request to api:\n",
    "    try:\n",
    "        crossref_api_request = session.get(\n",
    "            crossref_api_url, timeout=20\n",
    "        )\n",
    "    except TimeoutError:\n",
    "        print(\"Timeout!\")\n",
    "        return None\n",
    "    else:\n",
    "        # if the request was successful, get the json response:\n",
    "        crossref_api_response = crossref_api_request.json()\n",
    "        # print(crossref_api_response)\n",
    "        # check if there is a doi in the response:\n",
    "        try:\n",
    "            doi = crossref_api_response[\"message\"][\"items\"][0][\"DOI\"]\n",
    "        except:\n",
    "            return f\"DOI not found for {citation}\"\n",
    "        else:\n",
    "            # check if the title of the response matches the citation string (fuzzy match):\n",
    "            try:\n",
    "                title = crossref_api_response[\"message\"][\"items\"][0][\"title\"][0]\n",
    "            except:\n",
    "                return f\"Title not found for {citation}\"\n",
    "                title = None\n",
    "            else:\n",
    "                # use fuzzywuzzy:\n",
    "                # check if the title matches the citation string:\n",
    "                if fuzz.partial_ratio((title.lower()), clean_citation.lower()) < 50:\n",
    "                    return f\"Title mismatch: {title} != {clean_citation}\", None\n",
    "                else:\n",
    "                    return clean_citation, doi\n",
    "            return clean_citation, doi\n",
    "    \n",
    "for count ,citation in enumerate(rplics):\n",
    "    print(f\"{count+1}: {citation}\")\n",
    "    citation, doi = fetch_doi_from_citation_string(citation)\n",
    "    print(f\"{count+1}: {citation}: \\nhttps://doi.org/{doi}\")\n",
    "\n",
    "\n",
    "# so, this works, except for cases we need to handle:\n",
    "# - main field is not a citation string (or doi or url-doi), but a url (eg at osf.io)\n",
    "# - \"Testeintrag\"\n",
    "# - \"Stumbrys, T., & Erlacher, D. (2014). The science of lucid dream induction. In R. Hurd, & K. Bulkeley (Eds.). Lucid dreaming: New perspectives on consciousness in sleep (pp. 77–96). Westport: Praeger.:\"\n",
    "#   why that is: this is a chapter that doesn't have a doi. Just a record in APA PsycNet.\n",
    "# Conclusion: Catch any URLs and DOis (and the \"Testeintrag\") in the main field beforehand, save them as what they are.\n",
    "# Only look up \"real\" citations in crossref. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Graph identifier=N3727fc0dfdb8409ba168a777e2a90ea2 (<class 'rdflib.graph.Graph'>)>"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from rdflib import Graph, URIRef, Literal, Namespace, RDF, RDFS, BNode\n",
    "from rdflib.namespace import SKOS, OWL, DCTERMS, XSD\n",
    "\n",
    "# load the vocab as a graph:\n",
    "vocab = Graph()\n",
    "vocab.parse(\"terms_conversion/skosified_apa_thes_2023_02.ttl\", format=\"turtle\")\n",
    "\n",
    "list_of_excluded_concepts = [ \"https://w3id.org/zpid/vocabs/terms/00010\", \"https://w3id.org/zpid/vocabs/terms/00020\" ]\n",
    "# add a new triple:\n",
    "for block_concept in list_of_excluded_concepts:\n",
    "    for s,p,o in vocab.triples((URIRef(block_concept), RDF.type, SKOS.Concept)):\n",
    "        vocab.set((s, OWL.deprecated, Literal(\"true\", datatype=XSD.boolean)))\n",
    "        vocab.set((s, SKOS.editorialNote, Literal(\"annif-blocklisted\", datatype=XSD.string)))\n",
    "vocab.serialize(\"reduced_testvoc.ttl\", format=\"turtle\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-20\n",
      "2021-06-07\n",
      "2021\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "'NoneType' object has no attribute 'strftime'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[3], line 10\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[38;5;28mprint\u001b[39m(dateparser\u001b[38;5;241m.\u001b[39mparse(date2)\u001b[38;5;241m.\u001b[39mstrftime(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m%\u001b[39m\u001b[38;5;124mY-\u001b[39m\u001b[38;5;124m%\u001b[39m\u001b[38;5;124mm-\u001b[39m\u001b[38;5;132;01m%d\u001b[39;00m\u001b[38;5;124m\"\u001b[39m))\n\u001b[1;32m      9\u001b[0m \u001b[38;5;28mprint\u001b[39m(dateparser\u001b[38;5;241m.\u001b[39mparse(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m2021\u001b[39m\u001b[38;5;124m\"\u001b[39m)\u001b[38;5;241m.\u001b[39mstrftime(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m%\u001b[39m\u001b[38;5;124mY\u001b[39m\u001b[38;5;124m\"\u001b[39m))  \n\u001b[0;32m---> 10\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[43mdateparser\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparse\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m: 23. September 2021\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstrftime\u001b[49m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m%\u001b[39m\u001b[38;5;124mY-\u001b[39m\u001b[38;5;124m%\u001b[39m\u001b[38;5;124mm-\u001b[39m\u001b[38;5;132;01m%d\u001b[39;00m\u001b[38;5;124m\"\u001b[39m))\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'NoneType' object has no attribute 'strftime'"
     ]
    }
   ],
   "source": [
    "import datetime\n",
    "import dateparser\n",
    "\n",
    "date = \"20.10.2021\"\n",
    "date2 = \"7 June 2021\"\n",
    "\n",
    "print(dateparser.parse(date).strftime(\"%Y-%m-%d\"))\n",
    "print(dateparser.parse(date2).strftime(\"%Y-%m-%d\"))\n",
    "print(dateparser.parse(\"2021\").strftime(\"%Y\"))  \n",
    "print(dateparser.parse(\": 23. September 2021\").strftime(\"%Y-%m-%d\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "neuer abstract: Healthy mental functions are essentially based on undisturbed synaptic processes in the brain. Drugs that can potentially affect synapse formation, neurotransmitter metabolism, or action potentials carry a risk of leading to psychiatric disorders. In particular, ligands of neurotransmitter receptors as well as hormones have a (b) high psychiatric side effect potential. (c) Thieme. All rights reserved.\n",
      "lizenzstring: None\n"
     ]
    }
   ],
   "source": [
    "abstracts= 'Healthy mental functions are essentially based on undisturbed synaptic processes in the brain. Drugs that can potentially affect synapse formation, neurotransmitter metabolism, or action potentials carry a risk of leading to psychiatric disorders. In particular, ligands of neurotransmitter receptors as well as hormones have a (b) high psychiatric side effect potential. (c) Thieme. All rights reserved.'\n",
    "\n",
    "import re\n",
    "\n",
    "def add_abstract_licensing_note(abstracttext):\n",
    "    \"\"\"Adds a licensing note to the abstract if it contains a copyright string and/or a \"translated by DeepL\" notice.\"\"\"\n",
    "    abstract_copyright_string = None\n",
    "    # 1. first check if there is a \"(translated by DeepL)\" at the end of the abstract, remove it and add it to the licensing note.\n",
    "    # 2 then check for a copyright string at the (new) end of the abstract. Remove it and copy it into the licensinf note - \n",
    "    # but only if there isn't already something in there (the translated by deepl note) - because if there is, the translation note takes precedence\n",
    "    # and the copyright note will not be retained. \n",
    "    deepl_match = re.search(r\"^(.*)\\s\\((translated by DeepL)\\)$\", abstracttext, re.IGNORECASE)\n",
    "    if deepl_match:\n",
    "        # replace the abstract with the content before the \"(translated by DeepL)\":\n",
    "        abstracttext = deepl_match.group(1)\n",
    "        # add it to the licensing note, but only if empty:\n",
    "        abstract_copyright_string = deepl_match.group(2)\n",
    "    else:\n",
    "        abstract_copyright_string = None\n",
    "\n",
    "    # also, after that, check the new abstract for a copyright string:\n",
    "    license_match = re.search(r\"(.*)(\\(c\\).*)$\", abstracttext, re.IGNORECASE)\n",
    "    # if that match is not None, check if it is in the last 100 characters of the abstract:\n",
    "    if license_match and len(license_match.group(2)) < 100:\n",
    "        # if so, check if there is a \"(b)\" anywhere in the abstract before the match (this is an exclusion criterion,\n",
    "        # because if there is a \"(b)\" before the \"(c)\", it's just a lettered list item, not the copyright string):\n",
    "        if re.search(r\"(.*)(\\(b\\).*)\", license_match.group(1), re.IGNORECASE):\n",
    "            pass\n",
    "            # if there is _no_ \"(b)\" before the \"(c)\", we have a copyright string; add it to the licensing note.\n",
    "            # unless it already contains something - which will always be the translation note:\n",
    "        else:\n",
    "            if abstract_copyright_string is None or abstract_copyright_string == \"\":\n",
    "                abstract_copyright_string = license_match.group(2)\n",
    "                abstracttext = license_match.group(1)\n",
    "            else:\n",
    "                # don't write it into the note if there is already something in it, but do remove it from the abstract!\n",
    "                abstracttext = license_match.group(1)\n",
    "            # otherwise ignore the string, we have no copyright string\n",
    "    \n",
    "    print(\"neuer abstract: \" + abstracttext)\n",
    "    print(\"lizenzstring: \" + str(abstract_copyright_string))\n",
    "\n",
    "add_abstract_licensing_note(abstracts)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deciding Genre\n",
    "\n",
    "Fields to use:\n",
    "- BE\n",
    "- DT, DT2\n",
    "- CM\n",
    "- CF (for conference proceedings)?\n",
    "- BN, BNDI\n",
    "- DIDH\n",
    "\n",
    "## Theses:\n",
    "\n",
    "## cumulative thesis\n",
    "- BN starts with \"Kumulative\" (14x in 556.xml)\n",
    "\n",
    "### ThesisHabilitation\n",
    "- BE = SM (oder SH?)\n",
    "- DT = 01 (SM)\n",
    "- und/oder DIDH = \"Habil.Schr.\n",
    "- und/oder BN = \"Kumulative Habilitationsschrift\"\n",
    "\n",
    "### ThesisDoctoral: \n",
    "- BE = SH (15x) oder DT=61 (15x) oder DT2 = 61\n",
    "- und/oder DIDH = \"Diss.\"\n",
    "- und/oder BN starts with = \"Kumulative Dissertation\"\n",
    "\n",
    "\n",
    "Note for Roles and contributions:\n",
    "<BN>Gesprächsführung: Vorname Nachname</BN> -> Contribution with role interviewer hinten anfügen, wenn es nicht schon eine gibt. Sonst: fehlende Rolle nachtragen bzw ändern von Autor zu IN\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Genres of the work: ['habilitation thesis', 'cumulative thesis']\n",
      "Genres of the work: ['doctoral thesis', 'cumulative thesis']\n",
      "Genres of the work: ['doctoral thesis', 'cumulative thesis']\n",
      "Genres of the work: ['doctoral thesis']\n",
      "Genres of the work: ['doctoral thesis']\n"
     ]
    }
   ],
   "source": [
    "# to map any cm to a matching genre, do a search of the cm's code/notation in the genres vocid, and use the first hit. I used the same notations for genres that were taken from cm - so the api should find the genre that matches the cm.\n",
    "\n",
    "# also, to map any empirical cm to \"Research Paper\", go by number of the code, maybe? Anything starting with 10 is empirical.\n",
    "test_records = [\n",
    "    { \"dfk\":\"0227202\", \"be\":\"SM\", \"dt\":\"01\", \"dt2\":\"\", \"cm\":\"\", \"didh\":\"\", \"bn\":\"Kumulative Habilitationsschrift\", \"bndi\":\"\"},\n",
    "    { \"dfk\":\"0390734\", \"be\":\"SH\", \"dt\":\"61\", \"dt2\":\"\", \"cm\":\"\",\"didh\":\"\", \"bn\":\"Kumulative Dissertation\", \"bndi\":\"\"},\n",
    "    {\"dfk\": \"0017943\", \"be\":\"SH\",\"dt\":\"61\",\"dt2\":\"01\", \"cm\":\"\", \"didh\":\"Diss.\", \"bn\":\"Kumulative Dissertation: (1) Pavel, F.-G. 1978. Die klientenzentrierte Psychotherapie. München: Pfeiffer; (2) Beitrag in: Spiel, W. (Ed.) 1980. Die Psychologie des 20. Jahrhunderts. Bd. 12. Zürich: Kindler. S. 844-864; (3) GwG-Info 1982, 47, 37-48; (4) GwG-Info 1983, 52, 7-27; (5) Zeitschrift f. Personenzentr. Psychol. u. Psychotherapie 1984, 3, 277-300\", \"bndi\":\"\"},\n",
    "    {\n",
    "        \"dfk\": \"0017943\",\"be\":\"SH\", \"cm\":\"\", \"didh\":\"Diss.\", \"dt\":\"61\", \"dt2\":\"01\", \"bn\":\"Schreibmaschinenfassung\", \"bndi\":\"\"\n",
    "    },\n",
    "      {\n",
    "        \"dfk\": \"0017943\",\"be\":\"SH\", \"cm\":\"\", \"didh\":\"Diss.\", \"dt\":\"61\", \"dt2\":\"01\", \"bn\":\"Schreibmaschinenfassung\", \"bndi\":\"\"\n",
    "    },\n",
    "          {\n",
    "        \"dfk\": \"0017943\",\"be\":\"SH\", \"cm\":\"\", \"didh\":\"Diss.\", \"dt\":\"61\", \"dt2\":\"01\", \"bn\":\"Schreibmaschinenfassung\", \"bndi\":\"\"\n",
    "    }\n",
    "    ]\n",
    "\n",
    "def determine_genre(be, dt=\"\", dt2=\"\", cm=\"\", didh=\"\", bn=\"\", bndi=\"\"):\n",
    "    \n",
    "    genres = []\n",
    "    # doctoral thesis:\n",
    "    if be == \"SH\" or dt == \"61\" or dt2 == \"61\" or \"Diss\".casefold() in didh.casefold():\n",
    "        genres.append(\"doctoral thesis\")\n",
    "    # habilitation thesis:\n",
    "    if \"habil\".casefold() in didh.casefold() or \"habilitationsschrift\".casefold() in bn.casefold():\n",
    "        genres.append(\"habilitation thesis\")\n",
    "    # cumulative thesis:\n",
    "    if \"kumulativ\".casefold() in bn.casefold():\n",
    "         genres.append(\"cumulative thesis\")\n",
    "            \n",
    "    return genres\n",
    "\n",
    "for record in test_records:\n",
    "    print(\"Genres of the work: \" \n",
    "      + str(determine_genre(record[\"be\"], record[\"dt\"], record[\"dt2\"], record[\"cm\"], record[\"didh\"], record[\"bn\"], record[\"bndi\"])))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Code to fix weird dates in field PY\n",
    "\n",
    "https://docs.google.com/spreadsheets/d/1ESISF24A4QbXw8Vf7EAzRY1oaYYnkNVsYoz45J5Ri9Y/edit?gid=2136685722#gid=2136685722\n",
    "\n",
    "The Goal is to always use the first year that is given, if there are several. That can be hard because the year formatting can differ wildly.\n",
    "\n",
    "1993/94 -> 1993\n",
    "1998/99\n",
    "1999/2000 -> 1999\n",
    "2000/2001\n",
    "2003-2004\n",
    "2005-2006\n",
    "2001/2002\n",
    "2005-2006 -> 2005\n",
    "2014-2015 -> 2014\n",
    "2005-06 -> 2005\n",
    "2001-2002\n",
    "1981-83 -> 1981\n",
    "1981-83\n",
    "1981-83\n",
    "1981-83\n",
    "1981-83\n",
    "1981-83\n",
    "1979/80\n",
    "1982-83\n",
    "1982-83\n",
    "1982-83\n",
    "1982-83\n",
    "1982-83\n",
    "1982-83\n",
    "1982-83\n",
    "1976/77\n",
    "1986/87\n",
    "1986/87\n",
    "1986/87\n",
    "1986/87\n",
    "1986/87\n",
    "1986/87\n",
    "1985/86\n",
    "1985/86\n",
    "1985/86\n",
    "1985/86\n",
    "1986/87\n",
    "1986/87\n",
    "1986/87\n",
    "1986/87\n",
    "1986/87\n",
    "1986/87\n",
    "1986/87\n",
    "1986/87\n",
    "1986/87\n",
    "1986/87\n",
    "1986/87\n",
    "1986/87\n",
    "1986/87\n",
    "1988/89\n",
    "1988/89\n",
    "1988/89\n",
    "1988/89\n",
    "1988/89\n",
    "1988/89\n",
    "1988/89\n",
    "1988/89\n",
    "1989/90\n",
    "1989/90\n",
    "1989/90\n",
    "1989/90\n",
    "1989/90\n",
    "1989/90\n",
    "1989/90\n",
    "1989/90\n",
    "1989/90\n",
    "1989/90\n",
    "1989/90\n",
    "1989/90\n",
    "1989/90\n",
    "1989/90\n",
    "1990/91\n",
    "1990/91\n",
    "1990/91\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1998/99 becomes 1998\n",
      "1993/94 becomes 1993\n",
      "1999/2000 becomes 1999\n",
      "2000/2001 becomes 2000\n",
      "2003-2004 becomes 2003\n",
      "2005-2006 becomes 2005\n",
      "2001/2002 becomes 2001\n",
      "2005-2006 becomes 2005\n",
      "2014-2015 becomes 2014\n",
      "2005-06 becomes 2005\n",
      "2001-2002 becomes 2001\n",
      "1981-83 becomes 1981\n",
      "1981-83 becomes 1981\n",
      "1981-83 becomes 1981\n",
      "1981-83 becomes 1981\n",
      "1981-83 becomes 1981\n",
      "1981-83 becomes 1981\n",
      "1979/80 becomes 1979\n",
      "1982-83 becomes 1982\n",
      "1982-83 becomes 1982\n",
      "1982-83 becomes 1982\n",
      "1982-83 becomes 1982\n",
      "1982-83 becomes 1982\n",
      "1982-83 becomes 1982\n",
      "1982-83 becomes 1982\n",
      "1976/77 becomes 1976\n",
      "1986/87 becomes 1986\n",
      "1986/87 becomes 1986\n",
      "1986/87 becomes 1986\n",
      "1986/87 becomes 1986\n",
      "1986/87 becomes 1986\n",
      "1986/87 becomes 1986\n",
      "1985/86 becomes 1985\n",
      "1985/86 becomes 1985\n",
      "1985/86 becomes 1985\n",
      "1985/86 becomes 1985\n",
      "1986/87 becomes 1986\n",
      "1986/87 becomes 1986\n",
      "1986/87 becomes 1986\n",
      "1986/87 becomes 1986\n",
      "1986/87 becomes 1986\n",
      "1986/87 becomes 1986\n",
      "1986/87 becomes 1986\n",
      "1986/87 becomes 1986\n",
      "1986/87 becomes 1986\n",
      "1986/87 becomes 1986\n",
      "1986/87 becomes 1986\n",
      "1986/87 becomes 1986\n",
      "1986/87 becomes 1986\n",
      "1988/89 becomes 1988\n",
      "1988/89 becomes 1988\n",
      "1988/89 becomes 1988\n",
      "1988/89 becomes 1988\n",
      "1988/89 becomes 1988\n",
      "1988/89 becomes 1988\n",
      "1988/89 becomes 1988\n",
      "1988/89 becomes 1988\n",
      "1989/90 becomes 1989\n",
      "1989/90 becomes 1989\n",
      "1989/90 becomes 1989\n",
      "1989/90 becomes 1989\n",
      "1989/90 becomes 1989\n",
      "1989/90 becomes 1989\n",
      "1989/90 becomes 1989\n",
      "1989/90 becomes 1989\n",
      "1989/90 becomes 1989\n",
      "1989/90 becomes 1989\n",
      "1989/90 becomes 1989\n",
      "1989/90 becomes 1989\n",
      "1989/90 becomes 1989\n",
      "1989/90 becomes 1989\n",
      "1990/91 becomes 1990\n",
      "1990/91 becomes 1990\n",
      "1990/91 becomes 1990\n",
      "1990/91 becomes 1990\n",
      "1990/91 becomes 1990\n",
      "1990/91 becomes 1990\n",
      "1990/91 becomes 1990\n",
      "1990/91 becomes 1990\n",
      "1990/91 becomes 1990\n",
      "1990/91 becomes 1990\n",
      "1990/91 becomes 1990\n",
      "1991/92 becomes 1991\n",
      "1991/92 becomes 1991\n",
      "1991/92 becomes 1991\n",
      "1991/92 becomes 1991\n",
      "1991/92 becomes 1991\n",
      "1991/92 becomes 1991\n",
      "1991/92 becomes 1991\n",
      "1991/92 becomes 1991\n",
      "1990/91 becomes 1990\n",
      "1990/91 becomes 1990\n",
      "1990/91 becomes 1990\n",
      "1990/91 becomes 1990\n",
      "1990/91 becomes 1990\n",
      "1992/93 becomes 1992\n",
      "1991/92 becomes 1991\n",
      "1991/92 becomes 1991\n",
      "1991/92 becomes 1991\n",
      "1991/92 becomes 1991\n",
      "1991/92 becomes 1991\n",
      "1991/92 becomes 1991\n",
      "1991/92 becomes 1991\n",
      "1980-90 becomes 1980\n",
      "1980-90 becomes 1980\n",
      "1993/94 becomes 1993\n",
      "1993/94 becomes 1993\n",
      "1993/94 becomes 1993\n",
      "1993/94 becomes 1993\n",
      "1993/94 becomes 1993\n",
      "1993/94 becomes 1993\n",
      "1993/94 becomes 1993\n",
      "1993/94 becomes 1993\n",
      "1993/94 becomes 1993\n",
      "1993/94 becomes 1993\n",
      "1993/94 becomes 1993\n",
      "1993/94 becomes 1993\n",
      "1993/94 becomes 1993\n",
      "1993/94 becomes 1993\n",
      "1993/94 becomes 1993\n",
      "1993/94 becomes 1993\n",
      "1993/94 becomes 1993\n",
      "1993/94 becomes 1993\n",
      "1993/94 becomes 1993\n",
      "1993/94 becomes 1993\n",
      "1993/94 becomes 1993\n",
      "1985/86 becomes 1985\n",
      "1995/96 becomes 1995\n",
      "1991/92 becomes 1991\n",
      "1991/92 becomes 1991\n",
      "1991/92 becomes 1991\n",
      "1991/92 becomes 1991\n",
      "1991/92 becomes 1991\n",
      "1991/92 becomes 1991\n",
      "1995/96 becomes 1995\n",
      "1995/96 becomes 1995\n",
      "1995/96 becomes 1995\n",
      "1995/96 becomes 1995\n",
      "1995/96 becomes 1995\n",
      "1995/96 becomes 1995\n",
      "1995/96 becomes 1995\n",
      "1995/96 becomes 1995\n",
      "1995/96 becomes 1995\n",
      "1995/96 becomes 1995\n",
      "1995/96 becomes 1995\n",
      "1995/96 becomes 1995\n",
      "1995/96 becomes 1995\n",
      "1995/96 becomes 1995\n",
      "1995/96 becomes 1995\n",
      "1995/96 becomes 1995\n",
      "1995/96 becomes 1995\n",
      "1995/96 becomes 1995\n",
      "1995/96 becomes 1995\n",
      "1993/95 becomes 1993\n",
      "1993/95 becomes 1993\n",
      "1993/95 becomes 1993\n",
      "1993/95 becomes 1993\n",
      "1993/95 becomes 1993\n",
      "1996/97 becomes 1996\n",
      "1996/97 becomes 1996\n",
      "1996/97 becomes 1996\n",
      "1996/97 becomes 1996\n",
      "1996/97 becomes 1996\n",
      "1995/96 becomes 1995\n",
      "1996/97 becomes 1996\n",
      "1996/97 becomes 1996\n",
      "1993/95 becomes 1993\n",
      "1998/99 becomes 1998\n",
      "1990/91 becomes 1990\n",
      "1998/99 becomes 1998\n",
      "1999/2000 becomes 1999\n",
      "1999/2000 becomes 1999\n",
      "2000/2001 becomes 2000\n",
      "2000/2001 becomes 2000\n",
      "2000/2001 becomes 2000\n",
      "2000/2001 becomes 2000\n",
      "2000/2001 becomes 2000\n",
      "2000/2001 becomes 2000\n",
      "2000/2001 becomes 2000\n",
      "2000/2001 becomes 2000\n",
      "2000/2001 becomes 2000\n",
      "2000/2001 becomes 2000\n",
      "2000/2001 becomes 2000\n",
      "2000/2001 becomes 2000\n",
      "2000/2001 becomes 2000\n",
      "2001/2002 becomes 2001\n",
      "2001/2002 becomes 2001\n",
      "2001/2002 becomes 2001\n",
      "2001/2002 becomes 2001\n",
      "1949/50 becomes 1949\n",
      "1949/50 becomes 1949\n",
      "1949/50 becomes 1949\n",
      "1949/50 becomes 1949\n",
      "1949/50 becomes 1949\n",
      "1949/50 becomes 1949\n",
      "1949/50 becomes 1949\n",
      "1949/50 becomes 1949\n",
      "1949/50 becomes 1949\n",
      "1949/50 becomes 1949\n",
      "1949/50 becomes 1949\n",
      "1949/50 becomes 1949\n",
      "1949/50 becomes 1949\n",
      "1949/50 becomes 1949\n",
      "1949/50 becomes 1949\n",
      "1949/50 becomes 1949\n",
      "1949/50 becomes 1949\n",
      "1949/50 becomes 1949\n",
      "1949/50 becomes 1949\n",
      "1949/50 becomes 1949\n",
      "1949/50 becomes 1949\n",
      "1949/50 becomes 1949\n",
      "1949/50 becomes 1949\n",
      "1949/50 becomes 1949\n",
      "1949/50 becomes 1949\n",
      "1949/50 becomes 1949\n",
      "1949/50 becomes 1949\n",
      "1949/50 becomes 1949\n",
      "1949/50 becomes 1949\n",
      "2000/2001 becomes 2000\n",
      "2000/2001 becomes 2000\n",
      "2000/2001 becomes 2000\n",
      "2000/2001 becomes 2000\n",
      "2000/2001 becomes 2000\n",
      "2000/2001 becomes 2000\n",
      "1998/99 becomes 1998\n",
      "2007/2008 becomes 2007\n",
      "1988-1989 becomes 1988\n",
      "2009-2010 becomes 2009\n",
      "2009-2010 becomes 2009\n",
      "2009-2010 becomes 2009\n",
      "2009-2010 becomes 2009\n",
      "2009-2010 becomes 2009\n",
      "2009-2010 becomes 2009\n",
      "2009-2010 becomes 2009\n",
      "2009-2010 becomes 2009\n",
      "2009-2010 becomes 2009\n",
      "2008-2009 becomes 2008\n",
      "2008-2009 becomes 2008\n",
      "2008-2009 becomes 2008\n",
      "2008-2009 becomes 2008\n",
      "2008-2009 becomes 2008\n",
      "2008-2009 becomes 2008\n",
      "2008-2009 becomes 2008\n",
      "2005-2007 becomes 2005\n",
      "2005-2007 becomes 2005\n",
      "2005-2007 becomes 2005\n",
      "2005-2007 becomes 2005\n",
      "2005-2007 becomes 2005\n",
      "2005-2007 becomes 2005\n",
      "2005-2007 becomes 2005\n",
      "2005-2007 becomes 2005\n",
      "2005-2007 becomes 2005\n",
      "2011-2012 becomes 2011\n",
      "2011-2012 becomes 2011\n",
      "2011-2012 becomes 2011\n",
      "2011-2012 becomes 2011\n",
      "2011-2012 becomes 2011\n",
      "2011-2012 becomes 2011\n",
      "2011-2012 becomes 2011\n",
      "2011-2012 becomes 2011\n",
      "2011-2012 becomes 2011\n",
      "2011-2012 becomes 2011\n",
      "2011-2012 becomes 2011\n",
      "2002/03 becomes 2002\n",
      "2005/2006 becomes 2005\n",
      "2014-2015 becomes 2014\n",
      "2014-2015 becomes 2014\n",
      "2014-2015 becomes 2014\n",
      "2014-2015 becomes 2014\n",
      "2014-2015 becomes 2014\n",
      "2014-2015 becomes 2014\n",
      "2014-2015 becomes 2014\n",
      "2014-2015 becomes 2014\n",
      "2014-2015 becomes 2014\n",
      "2014-2015 becomes 2014\n",
      "2014-2015 becomes 2014\n",
      "2015-2016 becomes 2015\n",
      "2015-2016 becomes 2015\n",
      "2015-2016 becomes 2015\n",
      "2015-2016 becomes 2015\n",
      "2015-2016 becomes 2015\n",
      "2015-2016 becomes 2015\n",
      "2015-2016 becomes 2015\n",
      "2015-2016 becomes 2015\n",
      "2005-2006 becomes 2005\n",
      "2015-2016 becomes 2015\n",
      "2016-2017 becomes 2016\n",
      "2000/2001 becomes 2000\n",
      "2020-2021 becomes 2020\n",
      "2020-2021 becomes 2020\n",
      "2020-2021 becomes 2020\n",
      "2020-2021 becomes 2020\n",
      "2020-2021 becomes 2020\n",
      "2020-2021 becomes 2020\n",
      "2020-2021 becomes 2020\n",
      "2020-2021 becomes 2020\n",
      "2020-2021 becomes 2020\n",
      "2020-2021 becomes 2020\n",
      "2020-2021 becomes 2020\n",
      "2020-2021 becomes 2020\n",
      "2020-2021 becomes 2020\n",
      "2019-2020 becomes 2019\n",
      "2019-2020 becomes 2019\n",
      "2019-2020 becomes 2019\n",
      "22 becomes 4 digits, but not a valid year\n",
      "34 becomes 4 digits, but not a valid year\n",
      "o.J. becomes error: can't interpret year, please correct\n",
      "1671 becomes 4 digits, but not a valid year\n",
      "keine Angabe becomes error: can't interpret year, please correct\n",
      "abcd becomes error: can't interpret year, please correct\n",
      "1111 becomes 4 digits, but not a valid year\n"
     ]
    }
   ],
   "source": [
    "# list of PY years:\n",
    "PY = [\n",
    "    \"1998/99\",\n",
    "    \"1993/94\",\n",
    "    \"1999/2000\",\n",
    "    \"2000/2001\",\n",
    "    \"2003-2004\",\n",
    "    \"2005-2006\",\n",
    "    \"2001/2002\",\n",
    "    \"2005-2006\",\n",
    "    \"2014-2015\",\n",
    "    \"2005-06\",\n",
    "    \"2001-2002\",\n",
    "    \"1981-83\",\n",
    "    \"1981-83\",\n",
    "    \"1981-83\",\n",
    "    \"1981-83\",\n",
    "    \"1981-83\",\n",
    "    \"1981-83\",\n",
    "    \"1979/80\",\n",
    "    \"1982-83\",\n",
    "    \"1982-83\",\n",
    "    \"1982-83\",\n",
    "    \"1982-83\",\n",
    "    \"1982-83\",\n",
    "    \"1982-83\",\n",
    "    \"1982-83\",\n",
    "    \"1976/77\",\n",
    "    \"1986/87\",\n",
    "    \"1986/87\",\n",
    "    \"1986/87\",\n",
    "    \"1986/87\",\n",
    "    \"1986/87\",\n",
    "    \"1986/87\",\n",
    "    \"1985/86\",\n",
    "    \"1985/86\",\n",
    "    \"1985/86\",\n",
    "    \"1985/86\",\n",
    "    \"1986/87\",\n",
    "    \"1986/87\",\n",
    "    \"1986/87\",\n",
    "    \"1986/87\",\n",
    "    \"1986/87\",\n",
    "    \"1986/87\",\n",
    "    \"1986/87\",\n",
    "    \"1986/87\",\n",
    "    \"1986/87\",\n",
    "    \"1986/87\",\n",
    "    \"1986/87\",\n",
    "    \"1986/87\",\n",
    "    \"1986/87\",\n",
    "    \"1988/89\",\n",
    "    \"1988/89\",\n",
    "    \"1988/89\",\n",
    "    \"1988/89\",\n",
    "    \"1988/89\",\n",
    "    \"1988/89\",\n",
    "    \"1988/89\",\n",
    "    \"1988/89\",\n",
    "    \"1989/90\",\n",
    "    \"1989/90\",\n",
    "    \"1989/90\",\n",
    "    \"1989/90\",\n",
    "    \"1989/90\",\n",
    "    \"1989/90\",\n",
    "    \"1989/90\",\n",
    "    \"1989/90\",\n",
    "    \"1989/90\",\n",
    "    \"1989/90\",\n",
    "    \"1989/90\",\n",
    "    \"1989/90\",\n",
    "    \"1989/90\",\n",
    "    \"1989/90\",\n",
    "    \"1990/91\",\n",
    "    \"1990/91\",\n",
    "    \"1990/91\",\n",
    "    \"1990/91\",\n",
    "    \"1990/91\",\n",
    "    \"1990/91\",\n",
    "    \"1990/91\",\n",
    "    \"1990/91\",\n",
    "    \"1990/91\",\n",
    "    \"1990/91\",\n",
    "    \"1990/91\",\n",
    "    \"1991/92\",\n",
    "    \"1991/92\",\n",
    "    \"1991/92\",\n",
    "    \"1991/92\",\n",
    "    \"1991/92\",\n",
    "    \"1991/92\",\n",
    "    \"1991/92\",\n",
    "    \"1991/92\",\n",
    "    \"1990/91\",\n",
    "    \"1990/91\",\n",
    "    \"1990/91\",\n",
    "    \"1990/91\",\n",
    "    \"1990/91\",\n",
    "    \"1992/93\",\n",
    "    \"1991/92\",\n",
    "    \"1991/92\",\n",
    "    \"1991/92\",\n",
    "    \"1991/92\",\n",
    "    \"1991/92\",\n",
    "    \"1991/92\",\n",
    "    \"1991/92\",\n",
    "    \"1980-90\",\n",
    "    \"1980-90\",\n",
    "    \"1993/94\",\n",
    "    \"1993/94\",\n",
    "    \"1993/94\",\n",
    "    \"1993/94\",\n",
    "    \"1993/94\",\n",
    "    \"1993/94\",\n",
    "    \"1993/94\",\n",
    "    \"1993/94\",\n",
    "    \"1993/94\",\n",
    "    \"1993/94\",\n",
    "    \"1993/94\",\n",
    "    \"1993/94\",\n",
    "    \"1993/94\",\n",
    "    \"1993/94\",\n",
    "    \"1993/94\",\n",
    "    \"1993/94\",\n",
    "    \"1993/94\",\n",
    "    \"1993/94\",\n",
    "    \"1993/94\",\n",
    "    \"1993/94\",\n",
    "    \"1993/94\",\n",
    "    \"1985/86\",\n",
    "    \"1995/96\",\n",
    "    \"1991/92\",\n",
    "    \"1991/92\",\n",
    "    \"1991/92\",\n",
    "    \"1991/92\",\n",
    "    \"1991/92\",\n",
    "    \"1991/92\",\n",
    "    \"1995/96\",\n",
    "    \"1995/96\",\n",
    "    \"1995/96\",\n",
    "    \"1995/96\",\n",
    "    \"1995/96\",\n",
    "    \"1995/96\",\n",
    "    \"1995/96\",\n",
    "    \"1995/96\",\n",
    "    \"1995/96\",\n",
    "    \"1995/96\",\n",
    "    \"1995/96\",\n",
    "    \"1995/96\",\n",
    "    \"1995/96\",\n",
    "    \"1995/96\",\n",
    "    \"1995/96\",\n",
    "    \"1995/96\",\n",
    "    \"1995/96\",\n",
    "    \"1995/96\",\n",
    "    \"1995/96\",\n",
    "    \"1993/95\",\n",
    "    \"1993/95\",\n",
    "    \"1993/95\",\n",
    "    \"1993/95\",\n",
    "    \"1993/95\",\n",
    "    \"1996/97\",\n",
    "    \"1996/97\",\n",
    "    \"1996/97\",\n",
    "    \"1996/97\",\n",
    "    \"1996/97\",\n",
    "    \"1995/96\",\n",
    "    \"1996/97\",\n",
    "    \"1996/97\",\n",
    "    \"1993/95\",\n",
    "    \"1998/99\",\n",
    "    \"1990/91\",\n",
    "    \"1998/99\",\n",
    "    \"1999/2000\",\n",
    "    \"1999/2000\",\n",
    "    \"2000/2001\",\n",
    "    \"2000/2001\",\n",
    "    \"2000/2001\",\n",
    "    \"2000/2001\",\n",
    "    \"2000/2001\",\n",
    "    \"2000/2001\",\n",
    "    \"2000/2001\",\n",
    "    \"2000/2001\",\n",
    "    \"2000/2001\",\n",
    "    \"2000/2001\",\n",
    "    \"2000/2001\",\n",
    "    \"2000/2001\",\n",
    "    \"2000/2001\",\n",
    "    \"2001/2002\",\n",
    "    \"2001/2002\",\n",
    "    \"2001/2002\",\n",
    "    \"2001/2002\",\n",
    "    \"1949/50\",\n",
    "    \"1949/50\",\n",
    "    \"1949/50\",\n",
    "    \"1949/50\",\n",
    "    \"1949/50\",\n",
    "    \"1949/50\",\n",
    "    \"1949/50\",\n",
    "    \"1949/50\",\n",
    "    \"1949/50\",\n",
    "    \"1949/50\",\n",
    "    \"1949/50\",\n",
    "    \"1949/50\",\n",
    "    \"1949/50\",\n",
    "    \"1949/50\",\n",
    "    \"1949/50\",\n",
    "    \"1949/50\",\n",
    "    \"1949/50\",\n",
    "    \"1949/50\",\n",
    "    \"1949/50\",\n",
    "    \"1949/50\",\n",
    "    \"1949/50\",\n",
    "    \"1949/50\",\n",
    "    \"1949/50\",\n",
    "    \"1949/50\",\n",
    "    \"1949/50\",\n",
    "    \"1949/50\",\n",
    "    \"1949/50\",\n",
    "    \"1949/50\",\n",
    "    \"1949/50\",\n",
    "    \"2000/2001\",\n",
    "    \"2000/2001\",\n",
    "    \"2000/2001\",\n",
    "    \"2000/2001\",\n",
    "    \"2000/2001\",\n",
    "    \"2000/2001\",\n",
    "    \"1998/99\",\n",
    "    \"2007/2008\",\n",
    "    \"1988-1989\",\n",
    "    \"2009-2010\",\n",
    "    \"2009-2010\",\n",
    "    \"2009-2010\",\n",
    "    \"2009-2010\",\n",
    "    \"2009-2010\",\n",
    "    \"2009-2010\",\n",
    "    \"2009-2010\",\n",
    "    \"2009-2010\",\n",
    "    \"2009-2010\",\n",
    "    \"2008-2009\",\n",
    "    \"2008-2009\",\n",
    "    \"2008-2009\",\n",
    "    \"2008-2009\",\n",
    "    \"2008-2009\",\n",
    "    \"2008-2009\",\n",
    "    \"2008-2009\",\n",
    "    \"2005-2007\",\n",
    "    \"2005-2007\",\n",
    "    \"2005-2007\",\n",
    "    \"2005-2007\",\n",
    "    \"2005-2007\",\n",
    "    \"2005-2007\",\n",
    "    \"2005-2007\",\n",
    "    \"2005-2007\",\n",
    "    \"2005-2007\",\n",
    "    \"2011-2012\",\n",
    "    \"2011-2012\",\n",
    "    \"2011-2012\",\n",
    "    \"2011-2012\",\n",
    "    \"2011-2012\",\n",
    "    \"2011-2012\",\n",
    "    \"2011-2012\",\n",
    "    \"2011-2012\",\n",
    "    \"2011-2012\",\n",
    "    \"2011-2012\",\n",
    "    \"2011-2012\",\n",
    "    \"2002/03\",\n",
    "    \"2005/2006\",\n",
    "    \"2014-2015\",\n",
    "    \"2014-2015\",\n",
    "    \"2014-2015\",\n",
    "    \"2014-2015\",\n",
    "    \"2014-2015\",\n",
    "    \"2014-2015\",\n",
    "    \"2014-2015\",\n",
    "    \"2014-2015\",\n",
    "    \"2014-2015\",\n",
    "    \"2014-2015\",\n",
    "    \"2014-2015\",\n",
    "    \"2015-2016\",\n",
    "    \"2015-2016\",\n",
    "    \"2015-2016\",\n",
    "    \"2015-2016\",\n",
    "    \"2015-2016\",\n",
    "    \"2015-2016\",\n",
    "    \"2015-2016\",\n",
    "    \"2015-2016\",\n",
    "    \"2005-2006\",\n",
    "    \"2015-2016\",\n",
    "    \"2016-2017\",\n",
    "    \"2000/2001\",\n",
    "    \"2020-2021\",\n",
    "    \"2020-2021\",\n",
    "    \"2020-2021\",\n",
    "    \"2020-2021\",\n",
    "    \"2020-2021\",\n",
    "    \"2020-2021\",\n",
    "    \"2020-2021\",\n",
    "    \"2020-2021\",\n",
    "    \"2020-2021\",\n",
    "    \"2020-2021\",\n",
    "    \"2020-2021\",\n",
    "    \"2020-2021\",\n",
    "    \"2020-2021\",\n",
    "    \"2019-2020\",\n",
    "    \"2019-2020\",\n",
    "    \"2019-2020\",\n",
    "    \"22\",\"34\", # maybe we can try to make a year out of things like \"22\"-> 2022? (so anything that is not four digits, but two digits? what makes 22 different from 34, which would be invalid?)\n",
    "    \"o.J.\",\n",
    "    \"1671\",\n",
    "    \"keine Angabe\",\n",
    "    \"abcd\",\"1111\"\n",
    "]\n",
    "for year in PY:\n",
    "    print(year + \" becomes \", end=\"\")\n",
    "    try:\n",
    "        # get the first 4 characters of the string and convert to int:\n",
    "        # anything smaller than 1700 and larger than 2026 is not valid years:\n",
    "        if int(year[:4]) < 1700 or int(year[:4]) > 2026:\n",
    "                print(\"4 digits, but not a valid year\")\n",
    "        elif year[:4]:\n",
    "            print(year[:4])\n",
    "    except:\n",
    "        print(\"error: can't interpret year, please correct\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# code to convert finding aid for Adolf Würth archove collection urls to new ones\n",
    "http://www.awz.uni-wuerzburg.de/fileadmin/42050000/user_upload/Findbuecher/Findbuch_HvB-final-Mai_2017.pdf -> \n",
    "https://www.uni-wuerzburg.de/fileadmin/42050000/Findbuecher/Findbuch_HvB-final-Mai_2017.pdf\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.uni-wuerzburg.de/fileadmin/42050000/Findbuecher/Findbuch_HvB-final-Mai_2017.pdf\n",
      "https://example.org\n",
      "https://www.uni-wuerzburg.de/fileadmin/42050000/Findbuecher/FA_Lothar_Spillmann_29042016.pdf\n",
      "https://www.uni-wuerzburg.de/fileadmin/42050000/Findbuecher/FA_Friedrich_Sander_expansion_2015-1.pdf\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "urls = [\n",
    "    \"http://www.awz.uni-wuerzburg.de/fileadmin/42050000/user_upload/Findbuecher/Findbuch_HvB-final-Mai_2017.pdf\",\n",
    "    \"https://example.org\",\n",
    "    \"http://www.awz.uni-wuerzburg.de/fileadmin/42050000/user_upload/Findbuecher/FA_Lothar_Spillmann_29042016.pdf\", # here they also updated it, so the filename (since it is data-based) itself is broken and the new link will not work, either! But we can't catch that.\n",
    "\"http://www.awz.uni-wuerzburg.de/fileadmin/42050000/user_upload/Findbuecher/FA_Friedrich_Sander_expansion_2015-1.pdf\"\n",
    "    ]\n",
    "\n",
    "wuerth_pattern= re.compile(\"http:\\/\\/\\www\\.awz.uni-wuerzburg.de\\/fileadmin\\/([0-9]{8})\\/user_upload\")\n",
    "wuerth_replace = r\"https://www.uni-wuerzburg.de/fileadmin/\\1\"\n",
    "\n",
    "for url in urls:\n",
    "    print(re.sub(wuerth_pattern, wuerth_replace, url, flags=0))\n",
    "    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python-3.10.10",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}