PairResearchParser.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load libraries and global functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# analysis\n",
    "import collections\n",
    "import random\n",
    "import math\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data loading\n",
    "from copy import deepcopy\n",
    "from pymongo import MongoClient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# code performance\n",
    "import time\n",
    "import multiprocessing as mp\n",
    "from tqdm.notebook import tqdm as tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/gx/l659bw1n4cdd6fb6zmd8dnm80000gn/T/ipykernel_9508/3652881321.py:1: FutureWarning: Passing a negative integer is deprecated in version 1.0 and will not be supported in future version. Instead, use None to not limit the column width.\n",
      "  pd.set_option('display.max_colwidth', -1)\n"
     ]
    }
   ],
   "source": [
    "pd.set_option('display.max_colwidth', -1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fetch Pairing Data from [pairresearch.io](http://pairresearch.io/)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TODO: move URI to a .env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['affinities',\n",
       " 'objectlabs-system.admin.collections',\n",
       " 'tasks',\n",
       " 'meteor_accounts_loginServiceConfiguration',\n",
       " 'affinities_history',\n",
       " 'pairings',\n",
       " 'users',\n",
       " 'tasks_history',\n",
       " 'pairs_history',\n",
       " 'groups',\n",
       " 'objectlabs-system']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uri = 'mongodb+srv://admin:2CbyDEcZFeeVBYfH@pair-research.qaket.mongodb.net/pair-research?retryWrites=true&w=majority'\n",
    "dbName = 'pair-research'\n",
    "client = MongoClient(uri)\n",
    "db = client[dbName]\n",
    "db.list_collection_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Groups \n",
      "Original size: 780 --> New size: 129\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_id</th>\n",
       "      <th>groupName</th>\n",
       "      <th>description</th>\n",
       "      <th>creatorId</th>\n",
       "      <th>creatorName</th>\n",
       "      <th>creationDate</th>\n",
       "      <th>roles</th>\n",
       "      <th>members</th>\n",
       "      <th>active</th>\n",
       "      <th>activePairing</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2K3P9hP4GgHcCJf3R</td>\n",
       "      <td>Soraya Fereydooni</td>\n",
       "      <td>Heart failure</td>\n",
       "      <td>przk5z9KRdc9xHJvb</td>\n",
       "      <td>Open access</td>\n",
       "      <td>2020-07-14 16:01:51.070</td>\n",
       "      <td>[{'title': 'Professor', '_id': 'Sc2qeq4RMzNXc92fz'}, {'title': 'Post Doc', '_id': 'XetsebSEPPn9ev8im'}, {'title': 'Graduate Student', '_id': 'Gdi9eHiMekaX5i8f2'}, {'title': 'Undergraduate Student', '_id': 'QkiHN5rHeTTGgGSSE'}]</td>\n",
       "      <td>[{'fullName': 'Open access', 'userId': 'przk5z9KRdc9xHJvb', 'role': {'title': 'Professor', '_id': 'Sc2qeq4RMzNXc92fz'}, 'isAdmin': True, 'isPending': False}, {'fullName': 'soraya fereydooni', 'userId': 'pTccf3tKaykKAm7zC', 'role': {'title': 'Undergraduate Student', '_id': 'QkiHN5rHeTTGgGSSE'}, 'isAdmin': False, 'isPending': False}]</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2rFoGTfRa9LFdpQNA</td>\n",
       "      <td>DAPS Seminar 2017-2018</td>\n",
       "      <td>Making pairs for DAPS seminar student workshopping</td>\n",
       "      <td>tr36uYkNnruoNhix7</td>\n",
       "      <td>Katie Cheng</td>\n",
       "      <td>2017-08-22 17:17:11.968</td>\n",
       "      <td>[{'title': 'Professor', '_id': 'tJWwWP7CydqETYe92'}, {'title': 'Graduate Student', '_id': '68SwfM4MDrn5vMgiH'}, {'title': 'Undergraduate Student', '_id': 'vELdyxQoe4ji2BKez'}, {'_id': 'q3h6ixm2ZjSWhHAvf'}]</td>\n",
       "      <td>[{'fullName': 'Katie Cheng', 'userId': 'tr36uYkNnruoNhix7', 'role': {'title': 'Graduate Student', '_id': '68SwfM4MDrn5vMgiH'}, 'isAdmin': True, 'isPending': False}, {'fullName': 'Paulina Biernacki', 'userId': 'PWufwHDsbRaw4se4X', 'role': {'title': 'Graduate Student', '_id': '68SwfM4MDrn5vMgiH'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Glenn Davis', 'userId': 'kEZXdjhfohiGxJWdu', 'role': {'title': 'Graduate Student', '_id': '68SwfM4MDrn5vMgiH'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Philip Hernandez', 'userId': 'f8wwqTXaifkxxoAc2', 'role': {'title': 'Graduate Student', '_id': '68SwfM4MDrn5vMgiH'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Zainab Hosseini', 'userId': 'iyRaCwz7QzxPRSi5t', 'role': {'title': 'Graduate Student', '_id': '68SwfM4MDrn5vMgiH'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'cklam@stanford.edu', 'userId': 'vw5WcSqZ9RoxDnMKE', 'role': {'title': 'Graduate Student', '_id': '68SwfM4MDrn5vMgiH'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Judy Nguyen', 'userId': '3si95Pn6NjXTxCWcT', 'role': {'title': 'Graduate Student', '_id': '68SwfM4MDrn5vMgiH'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Rose Pozos', 'userId': 'wjKjXbr62b6A9GJBM', 'role': {'title': 'Graduate Student', '_id': '68SwfM4MDrn5vMgiH'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Ana Saavedra', 'userId': 'zjGgFgeCZC4HxLHsv', 'role': {'title': 'Graduate Student', '_id': '68SwfM4MDrn5vMgiH'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Krystal A Smith', 'userId': 'oHaB5YdCnMpPGkWc9', 'role': {'title': 'Graduate Student', '_id': '68SwfM4MDrn5vMgiH'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Ben Stenhaug', 'userId': 'CDou3AFzKSYrdu4DE', 'role': {'title': 'Graduate Student', '_id': '68SwfM4MDrn5vMgiH'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Emily Morton', 'userId': 'GLTz7m8y7RqZCYzxx', 'role': {'title': 'Graduate Student', '_id': '68SwfM4MDrn5vMgiH'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Veronica Lin', 'userId': 'piEcGDpz5pQ5ewGSk', 'role': {'title': 'Graduate Student', '_id': '68SwfM4MDrn5vMgiH'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Katie Remington', 'userId': 'HBdsTKcPMmtQJDYih', 'role': {'title': 'Professor', '_id': 'tJWwWP7CydqETYe92'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'brucemc@stanford.edu', 'userId': '3jnGNWwFnB2pNQCMA', 'role': {'_id': '55555555555555555', 'title': 'Pending'}, 'isAdmin': False, 'isPending': True}, {'fullName': 'Karen Wang', 'userId': '4GTGJJsYCSNy3FRkn', 'role': {'title': 'Graduate Student', '_id': '68SwfM4MDrn5vMgiH'}, 'isAdmin': False, 'isPending': False}]</td>\n",
       "      <td>True</td>\n",
       "      <td>SpiKfuqCoEZRLfDNK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2sikqRCzCtNPfynTq</td>\n",
       "      <td>Design of Playful Learning Environments</td>\n",
       "      <td>Grad seminar in the Learning Sciences</td>\n",
       "      <td>CQhpnQnrAQMZhD9tM</td>\n",
       "      <td>Nell O'Rourke</td>\n",
       "      <td>2019-09-30 15:14:25.187</td>\n",
       "      <td>[{'title': 'Professor', '_id': 'Pb67Cd6shwfMEHKEu'}, {'title': 'Post Doc', '_id': 'uXmsdtT6ZrkuLr4fE'}, {'title': 'Graduate Student', '_id': '8Su87Zr6qvmbGSqGg'}, {'title': 'Undergraduate Student', '_id': 'oDSCmjahHS3RBiDpv'}]</td>\n",
       "      <td>[{'fullName': 'Nell O'Rourke', 'userId': 'CQhpnQnrAQMZhD9tM', 'role': {'title': 'Professor', '_id': 'Pb67Cd6shwfMEHKEu'}, 'isAdmin': True, 'isPending': False}, {'fullName': 'BrianAndrus2023@u.northwestern.edu', 'userId': 'a2hJS7Ky9iZueeAuE', 'role': {'title': 'Graduate Student', '_id': '8Su87Zr6qvmbGSqGg'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Wade Berger', 'userId': 'pMi2REP2EFpKMwyCo', 'role': {'title': 'Graduate Student', '_id': '8Su87Zr6qvmbGSqGg'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Garrett Hedman', 'userId': 'jjbvfDJxsyFDDGAB6', 'role': {'title': 'Graduate Student', '_id': '8Su87Zr6qvmbGSqGg'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'NicholasLagrassa2023@u.northwestern.edu', 'userId': 'NvftYR8DacP63P9sy', 'role': {'title': 'Graduate Student', '_id': '8Su87Zr6qvmbGSqGg'}, 'isAdmin': False, 'isPending': True}, {'fullName': 'Sarah', 'userId': 'v47hJxP5ZgTfMbbEH', 'role': {'title': 'Graduate Student', '_id': '8Su87Zr6qvmbGSqGg'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Alexis', 'userId': 'E4X7TZqTPwgqFvoWv', 'role': {'title': 'Graduate Student', '_id': '8Su87Zr6qvmbGSqGg'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Apoorva Shivaram', 'userId': 'bedXSjTfsRSW3tGKv', 'role': {'title': 'Graduate Student', '_id': '8Su87Zr6qvmbGSqGg'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Vanessa Stewart', 'userId': '8mrvaYyXbyyWt9kR5', 'role': {'title': 'Graduate Student', '_id': '8Su87Zr6qvmbGSqGg'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Nick LaGrassa', 'userId': '7yemYfg5RHGKux8YH', 'role': {'title': 'Graduate Student', '_id': '8Su87Zr6qvmbGSqGg'}, 'isAdmin': False, 'isPending': False}]</td>\n",
       "      <td>True</td>\n",
       "      <td>qhZxmF7byXJszAQ7w</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2uDCYsTKttn4zuEY9</td>\n",
       "      <td>POEM Lab</td>\n",
       "      <td>Research group at NCSU</td>\n",
       "      <td>XXHPSMLDh75odCZQp</td>\n",
       "      <td>Chris Martens</td>\n",
       "      <td>2018-01-23 18:58:35.624</td>\n",
       "      <td>[{'title': 'Professor', '_id': '6zcLDqjRofjDY5y7Z'}, {'title': 'Postdoc', '_id': 'hjJtBetzcbvivnbqm'}, {'title': 'Graduate student', '_id': 'YjS6SucZ67j9LTdoX'}, {'title': 'Undergraduate student', '_id': 'sBxzkA563KvDWM8Cx'}, {'title': 'Other collaborator', '_id': 'QwNNXjKr544X5B2jd'}]</td>\n",
       "      <td>[{'fullName': 'Chris Martens', 'userId': 'XXHPSMLDh75odCZQp', 'role': {'title': 'Professor', '_id': '6zcLDqjRofjDY5y7Z'}, 'isAdmin': True, 'isPending': False}, {'fullName': 'Owais Iqbal', 'userId': 'FSAwMreJa4hrQqtbf', 'role': {'title': 'Graduate student', '_id': 'YjS6SucZ67j9LTdoX'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Markus Eger', 'userId': 'yG8bfvM8i9WbauqMF', 'role': {'title': 'Graduate student', '_id': 'YjS6SucZ67j9LTdoX'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Hannah Morrison', 'userId': 'byouPddT89m4ZpSmq', 'role': {'title': 'Undergraduate student', '_id': 'sBxzkA563KvDWM8Cx'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Alexander Card', 'userId': 'ReCiBgDY8t2BMwMrY', 'role': {'title': 'Graduate student', '_id': 'YjS6SucZ67j9LTdoX'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Sasha Azad', 'userId': '7Lhayh8GS6hDm7yKn', 'role': {'title': 'Graduate student', '_id': 'YjS6SucZ67j9LTdoX'}, 'isAdmin': False, 'isPending': False}]</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3Fhz2DZHy6ofre3Nr</td>\n",
       "      <td>ThinkCERCA Product</td>\n",
       "      <td>Where team members can come together and solve problems.</td>\n",
       "      <td>PhM7jWEpcs4DTf4xF</td>\n",
       "      <td>Arielle</td>\n",
       "      <td>2018-03-12 22:02:53.565</td>\n",
       "      <td>[{'title': 'Professor', '_id': '5cDrPWYqyeLCuavHp'}, {'title': 'Post Doc', '_id': 'TgD64DSnQojhGWayS'}, {'title': 'Graduate Student', '_id': 'ZKKp4xMqbMopm7Di8'}, {'title': 'Undergraduate Student', '_id': 'PEmXP5ZyReLH76FkZ'}, {'title': 'Developer', '_id': 'vT85KxNuBTyp4Jf6P'}, {'title': 'Designer', '_id': 'ffPaot3WSiSfDQmAE'}, {'title': 'Product Management', '_id': 'wTcFSLSR5YW7bzHcS'}]</td>\n",
       "      <td>[{'fullName': 'Arielle', 'userId': 'PhM7jWEpcs4DTf4xF', 'role': {'title': 'Designer', '_id': 'ffPaot3WSiSfDQmAE'}, 'isAdmin': True, 'isPending': False}, {'fullName': 'Akili Lee', 'userId': 'Jcpvg7sgsyHX3CD3A', 'role': {'title': 'Product Management', '_id': 'wTcFSLSR5YW7bzHcS'}, 'isAdmin': True, 'isPending': False}, {'fullName': 'Aislinn Grigas', 'userId': 'ryzov9uQkSYhoiHAp', 'role': {'title': 'Designer', '_id': 'ffPaot3WSiSfDQmAE'}, 'isAdmin': True, 'isPending': False}, {'fullName': 'abby@thinkcerca.com', 'userId': 'DaPbdignjw4TicKWM', 'role': {'_id': '55555555555555555', 'title': 'Pending'}, 'isAdmin': True, 'isPending': True}, {'fullName': 'Nelson Galvan', 'userId': 'pysieJsgZCN8jyPsq', 'role': {'title': 'Product Management', '_id': 'wTcFSLSR5YW7bzHcS'}, 'isAdmin': True, 'isPending': False}, {'fullName': 'Sophie Lin', 'userId': 'MppxEpn8Reykp5YkZ', 'role': {'title': 'Designer', '_id': 'ffPaot3WSiSfDQmAE'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'elisabeth.klann@thinkcerca.com', 'userId': '77Sk2FJzejmHcAvh7', 'role': {'_id': '55555555555555555', 'title': 'Pending'}, 'isAdmin': False, 'isPending': True}, {'fullName': 'tousue@thinkcerca.com', 'userId': '8bxEu7oHA2KmPxKqW', 'role': {'title': 'Designer', '_id': 'ffPaot3WSiSfDQmAE'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Ryan Lammers', 'userId': 'jd8EbQ2if8oY5yNc8', 'role': {'title': 'Designer', '_id': 'ffPaot3WSiSfDQmAE'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Rose Tarullo', 'userId': 'qPuxcFbj8iXeWg9gS', 'role': {'title': 'Designer', '_id': 'ffPaot3WSiSfDQmAE'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Manu', 'userId': 'uuXoKJbZgF2CSCsqB', 'role': {'title': 'Developer', '_id': 'vT85KxNuBTyp4Jf6P'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'alejandra@thinkcerca.com', 'userId': '4iGgaouLtFqdiLjrj', 'role': {'_id': '55555555555555555', 'title': 'Pending'}, 'isAdmin': False, 'isPending': True}, {'fullName': 'facundo@thinkcerca.com', 'userId': 'njhzGs77RwASiGt9i', 'role': {'_id': '55555555555555555', 'title': 'Pending'}, 'isAdmin': False, 'isPending': True}, {'fullName': 'Florencia Arnaiz', 'userId': 'vqHbmezhrcLpu35jt', 'role': {'title': 'Developer', '_id': 'vT85KxNuBTyp4Jf6P'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Jake Fishbain', 'userId': 'wsxwd2GT9LnRMknTk', 'role': {'title': 'Developer', '_id': 'vT85KxNuBTyp4Jf6P'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'jeramy@thinkcerca.com', 'userId': '9bSj3FgWFTa3gFt5w', 'role': {'title': 'Developer', '_id': 'vT85KxNuBTyp4Jf6P'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Lauren Miller', 'userId': 'fbLZ8vNDwT6Xx2pSL', 'role': {'title': 'Undergraduate Student', '_id': 'PEmXP5ZyReLH76FkZ'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Leandro', 'userId': 'dpP5oK62XD8SaWEZy', 'role': {'title': 'Developer', '_id': 'vT85KxNuBTyp4Jf6P'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Pablo Oldani', 'userId': 'tx9wovn3Bt7AgDiCB', 'role': {'title': 'Developer', '_id': 'vT85KxNuBTyp4Jf6P'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Sam Beam', 'userId': 'YEFRoDoqsKpKX48J2', 'role': {'title': 'Developer', '_id': 'vT85KxNuBTyp4Jf6P'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Zan Rosenthal', 'userId': 'Funz26f4TWygXPe3N', 'role': {'title': 'Developer', '_id': 'vT85KxNuBTyp4Jf6P'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Byron Matto', 'userId': 'z8J27d3TWsJrYTHKy', 'role': {'title': 'Developer', '_id': 'vT85KxNuBTyp4Jf6P'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Melissa Patterson', 'userId': 'qHqXKZ7kTMrpRZv54', 'role': {'title': 'Developer', '_id': 'vT85KxNuBTyp4Jf6P'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Phil', 'userId': 'kWtmktznBCdLFAdW8', 'role': {'title': 'Developer', '_id': 'vT85KxNuBTyp4Jf6P'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Stacy Mullins', 'userId': 'tTnxrPytbiaerDT4M', 'role': {'title': 'Developer', '_id': 'vT85KxNuBTyp4Jf6P'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'aislinn@thinkcerca.com', 'userId': '92Y7kEDJfej9xdats', 'role': {'_id': '55555555555555555', 'title': 'Pending'}, 'isAdmin': False, 'isPending': True}]</td>\n",
       "      <td>True</td>\n",
       "      <td>F73t2pJjaXSadWkv8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 _id                                groupName  \\\n",
       "0  2K3P9hP4GgHcCJf3R  Soraya Fereydooni                         \n",
       "1  2rFoGTfRa9LFdpQNA  DAPS Seminar 2017-2018                    \n",
       "2  2sikqRCzCtNPfynTq  Design of Playful Learning Environments   \n",
       "3  2uDCYsTKttn4zuEY9  POEM Lab                                  \n",
       "4  3Fhz2DZHy6ofre3Nr  ThinkCERCA Product                        \n",
       "\n",
       "                                                description  \\\n",
       "0  Heart failure                                              \n",
       "1  Making pairs for DAPS seminar student workshopping         \n",
       "2  Grad seminar in the Learning Sciences                      \n",
       "3  Research group at NCSU                                     \n",
       "4  Where team members can come together and solve problems.   \n",
       "\n",
       "           creatorId    creatorName            creationDate  \\\n",
       "0  przk5z9KRdc9xHJvb  Open access   2020-07-14 16:01:51.070   \n",
       "1  tr36uYkNnruoNhix7  Katie Cheng   2017-08-22 17:17:11.968   \n",
       "2  CQhpnQnrAQMZhD9tM  Nell O'Rourke 2019-09-30 15:14:25.187   \n",
       "3  XXHPSMLDh75odCZQp  Chris Martens 2018-01-23 18:58:35.624   \n",
       "4  PhM7jWEpcs4DTf4xF  Arielle       2018-03-12 22:02:53.565   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                    roles  \\\n",
       "0  [{'title': 'Professor', '_id': 'Sc2qeq4RMzNXc92fz'}, {'title': 'Post Doc', '_id': 'XetsebSEPPn9ev8im'}, {'title': 'Graduate Student', '_id': 'Gdi9eHiMekaX5i8f2'}, {'title': 'Undergraduate Student', '_id': 'QkiHN5rHeTTGgGSSE'}]                                                                                                                                                                       \n",
       "1  [{'title': 'Professor', '_id': 'tJWwWP7CydqETYe92'}, {'title': 'Graduate Student', '_id': '68SwfM4MDrn5vMgiH'}, {'title': 'Undergraduate Student', '_id': 'vELdyxQoe4ji2BKez'}, {'_id': 'q3h6ixm2ZjSWhHAvf'}]                                                                                                                                                                                            \n",
       "2  [{'title': 'Professor', '_id': 'Pb67Cd6shwfMEHKEu'}, {'title': 'Post Doc', '_id': 'uXmsdtT6ZrkuLr4fE'}, {'title': 'Graduate Student', '_id': '8Su87Zr6qvmbGSqGg'}, {'title': 'Undergraduate Student', '_id': 'oDSCmjahHS3RBiDpv'}]                                                                                                                                                                       \n",
       "3  [{'title': 'Professor', '_id': '6zcLDqjRofjDY5y7Z'}, {'title': 'Postdoc', '_id': 'hjJtBetzcbvivnbqm'}, {'title': 'Graduate student', '_id': 'YjS6SucZ67j9LTdoX'}, {'title': 'Undergraduate student', '_id': 'sBxzkA563KvDWM8Cx'}, {'title': 'Other collaborator', '_id': 'QwNNXjKr544X5B2jd'}]                                                                                                           \n",
       "4  [{'title': 'Professor', '_id': '5cDrPWYqyeLCuavHp'}, {'title': 'Post Doc', '_id': 'TgD64DSnQojhGWayS'}, {'title': 'Graduate Student', '_id': 'ZKKp4xMqbMopm7Di8'}, {'title': 'Undergraduate Student', '_id': 'PEmXP5ZyReLH76FkZ'}, {'title': 'Developer', '_id': 'vT85KxNuBTyp4Jf6P'}, {'title': 'Designer', '_id': 'ffPaot3WSiSfDQmAE'}, {'title': 'Product Management', '_id': 'wTcFSLSR5YW7bzHcS'}]   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             members  \\\n",
       "0  [{'fullName': 'Open access', 'userId': 'przk5z9KRdc9xHJvb', 'role': {'title': 'Professor', '_id': 'Sc2qeq4RMzNXc92fz'}, 'isAdmin': True, 'isPending': False}, {'fullName': 'soraya fereydooni', 'userId': 'pTccf3tKaykKAm7zC', 'role': {'title': 'Undergraduate Student', '_id': 'QkiHN5rHeTTGgGSSE'}, 'isAdmin': False, 'isPending': False}]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       \n",
       "1  [{'fullName': 'Katie Cheng', 'userId': 'tr36uYkNnruoNhix7', 'role': {'title': 'Graduate Student', '_id': '68SwfM4MDrn5vMgiH'}, 'isAdmin': True, 'isPending': False}, {'fullName': 'Paulina Biernacki', 'userId': 'PWufwHDsbRaw4se4X', 'role': {'title': 'Graduate Student', '_id': '68SwfM4MDrn5vMgiH'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Glenn Davis', 'userId': 'kEZXdjhfohiGxJWdu', 'role': {'title': 'Graduate Student', '_id': '68SwfM4MDrn5vMgiH'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Philip Hernandez', 'userId': 'f8wwqTXaifkxxoAc2', 'role': {'title': 'Graduate Student', '_id': '68SwfM4MDrn5vMgiH'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Zainab Hosseini', 'userId': 'iyRaCwz7QzxPRSi5t', 'role': {'title': 'Graduate Student', '_id': '68SwfM4MDrn5vMgiH'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'cklam@stanford.edu', 'userId': 'vw5WcSqZ9RoxDnMKE', 'role': {'title': 'Graduate Student', '_id': '68SwfM4MDrn5vMgiH'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Judy Nguyen', 'userId': '3si95Pn6NjXTxCWcT', 'role': {'title': 'Graduate Student', '_id': '68SwfM4MDrn5vMgiH'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Rose Pozos', 'userId': 'wjKjXbr62b6A9GJBM', 'role': {'title': 'Graduate Student', '_id': '68SwfM4MDrn5vMgiH'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Ana Saavedra', 'userId': 'zjGgFgeCZC4HxLHsv', 'role': {'title': 'Graduate Student', '_id': '68SwfM4MDrn5vMgiH'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Krystal A Smith', 'userId': 'oHaB5YdCnMpPGkWc9', 'role': {'title': 'Graduate Student', '_id': '68SwfM4MDrn5vMgiH'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Ben Stenhaug', 'userId': 'CDou3AFzKSYrdu4DE', 'role': {'title': 'Graduate Student', '_id': '68SwfM4MDrn5vMgiH'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Emily Morton', 'userId': 'GLTz7m8y7RqZCYzxx', 'role': {'title': 'Graduate Student', '_id': '68SwfM4MDrn5vMgiH'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Veronica Lin', 'userId': 'piEcGDpz5pQ5ewGSk', 'role': {'title': 'Graduate Student', '_id': '68SwfM4MDrn5vMgiH'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Katie Remington', 'userId': 'HBdsTKcPMmtQJDYih', 'role': {'title': 'Professor', '_id': 'tJWwWP7CydqETYe92'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'brucemc@stanford.edu', 'userId': '3jnGNWwFnB2pNQCMA', 'role': {'_id': '55555555555555555', 'title': 'Pending'}, 'isAdmin': False, 'isPending': True}, {'fullName': 'Karen Wang', 'userId': '4GTGJJsYCSNy3FRkn', 'role': {'title': 'Graduate Student', '_id': '68SwfM4MDrn5vMgiH'}, 'isAdmin': False, 'isPending': False}]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             \n",
       "2  [{'fullName': 'Nell O'Rourke', 'userId': 'CQhpnQnrAQMZhD9tM', 'role': {'title': 'Professor', '_id': 'Pb67Cd6shwfMEHKEu'}, 'isAdmin': True, 'isPending': False}, {'fullName': 'BrianAndrus2023@u.northwestern.edu', 'userId': 'a2hJS7Ky9iZueeAuE', 'role': {'title': 'Graduate Student', '_id': '8Su87Zr6qvmbGSqGg'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Wade Berger', 'userId': 'pMi2REP2EFpKMwyCo', 'role': {'title': 'Graduate Student', '_id': '8Su87Zr6qvmbGSqGg'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Garrett Hedman', 'userId': 'jjbvfDJxsyFDDGAB6', 'role': {'title': 'Graduate Student', '_id': '8Su87Zr6qvmbGSqGg'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'NicholasLagrassa2023@u.northwestern.edu', 'userId': 'NvftYR8DacP63P9sy', 'role': {'title': 'Graduate Student', '_id': '8Su87Zr6qvmbGSqGg'}, 'isAdmin': False, 'isPending': True}, {'fullName': 'Sarah', 'userId': 'v47hJxP5ZgTfMbbEH', 'role': {'title': 'Graduate Student', '_id': '8Su87Zr6qvmbGSqGg'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Alexis', 'userId': 'E4X7TZqTPwgqFvoWv', 'role': {'title': 'Graduate Student', '_id': '8Su87Zr6qvmbGSqGg'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Apoorva Shivaram', 'userId': 'bedXSjTfsRSW3tGKv', 'role': {'title': 'Graduate Student', '_id': '8Su87Zr6qvmbGSqGg'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Vanessa Stewart', 'userId': '8mrvaYyXbyyWt9kR5', 'role': {'title': 'Graduate Student', '_id': '8Su87Zr6qvmbGSqGg'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Nick LaGrassa', 'userId': '7yemYfg5RHGKux8YH', 'role': {'title': 'Graduate Student', '_id': '8Su87Zr6qvmbGSqGg'}, 'isAdmin': False, 'isPending': False}]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   \n",
       "3  [{'fullName': 'Chris Martens', 'userId': 'XXHPSMLDh75odCZQp', 'role': {'title': 'Professor', '_id': '6zcLDqjRofjDY5y7Z'}, 'isAdmin': True, 'isPending': False}, {'fullName': 'Owais Iqbal', 'userId': 'FSAwMreJa4hrQqtbf', 'role': {'title': 'Graduate student', '_id': 'YjS6SucZ67j9LTdoX'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Markus Eger', 'userId': 'yG8bfvM8i9WbauqMF', 'role': {'title': 'Graduate student', '_id': 'YjS6SucZ67j9LTdoX'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Hannah Morrison', 'userId': 'byouPddT89m4ZpSmq', 'role': {'title': 'Undergraduate student', '_id': 'sBxzkA563KvDWM8Cx'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Alexander Card', 'userId': 'ReCiBgDY8t2BMwMrY', 'role': {'title': 'Graduate student', '_id': 'YjS6SucZ67j9LTdoX'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Sasha Azad', 'userId': '7Lhayh8GS6hDm7yKn', 'role': {'title': 'Graduate student', '_id': 'YjS6SucZ67j9LTdoX'}, 'isAdmin': False, 'isPending': False}]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 \n",
       "4  [{'fullName': 'Arielle', 'userId': 'PhM7jWEpcs4DTf4xF', 'role': {'title': 'Designer', '_id': 'ffPaot3WSiSfDQmAE'}, 'isAdmin': True, 'isPending': False}, {'fullName': 'Akili Lee', 'userId': 'Jcpvg7sgsyHX3CD3A', 'role': {'title': 'Product Management', '_id': 'wTcFSLSR5YW7bzHcS'}, 'isAdmin': True, 'isPending': False}, {'fullName': 'Aislinn Grigas', 'userId': 'ryzov9uQkSYhoiHAp', 'role': {'title': 'Designer', '_id': 'ffPaot3WSiSfDQmAE'}, 'isAdmin': True, 'isPending': False}, {'fullName': 'abby@thinkcerca.com', 'userId': 'DaPbdignjw4TicKWM', 'role': {'_id': '55555555555555555', 'title': 'Pending'}, 'isAdmin': True, 'isPending': True}, {'fullName': 'Nelson Galvan', 'userId': 'pysieJsgZCN8jyPsq', 'role': {'title': 'Product Management', '_id': 'wTcFSLSR5YW7bzHcS'}, 'isAdmin': True, 'isPending': False}, {'fullName': 'Sophie Lin', 'userId': 'MppxEpn8Reykp5YkZ', 'role': {'title': 'Designer', '_id': 'ffPaot3WSiSfDQmAE'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'elisabeth.klann@thinkcerca.com', 'userId': '77Sk2FJzejmHcAvh7', 'role': {'_id': '55555555555555555', 'title': 'Pending'}, 'isAdmin': False, 'isPending': True}, {'fullName': 'tousue@thinkcerca.com', 'userId': '8bxEu7oHA2KmPxKqW', 'role': {'title': 'Designer', '_id': 'ffPaot3WSiSfDQmAE'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Ryan Lammers', 'userId': 'jd8EbQ2if8oY5yNc8', 'role': {'title': 'Designer', '_id': 'ffPaot3WSiSfDQmAE'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Rose Tarullo', 'userId': 'qPuxcFbj8iXeWg9gS', 'role': {'title': 'Designer', '_id': 'ffPaot3WSiSfDQmAE'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Manu', 'userId': 'uuXoKJbZgF2CSCsqB', 'role': {'title': 'Developer', '_id': 'vT85KxNuBTyp4Jf6P'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'alejandra@thinkcerca.com', 'userId': '4iGgaouLtFqdiLjrj', 'role': {'_id': '55555555555555555', 'title': 'Pending'}, 'isAdmin': False, 'isPending': True}, {'fullName': 'facundo@thinkcerca.com', 'userId': 'njhzGs77RwASiGt9i', 'role': {'_id': '55555555555555555', 'title': 'Pending'}, 'isAdmin': False, 'isPending': True}, {'fullName': 'Florencia Arnaiz', 'userId': 'vqHbmezhrcLpu35jt', 'role': {'title': 'Developer', '_id': 'vT85KxNuBTyp4Jf6P'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Jake Fishbain', 'userId': 'wsxwd2GT9LnRMknTk', 'role': {'title': 'Developer', '_id': 'vT85KxNuBTyp4Jf6P'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'jeramy@thinkcerca.com', 'userId': '9bSj3FgWFTa3gFt5w', 'role': {'title': 'Developer', '_id': 'vT85KxNuBTyp4Jf6P'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Lauren Miller', 'userId': 'fbLZ8vNDwT6Xx2pSL', 'role': {'title': 'Undergraduate Student', '_id': 'PEmXP5ZyReLH76FkZ'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Leandro', 'userId': 'dpP5oK62XD8SaWEZy', 'role': {'title': 'Developer', '_id': 'vT85KxNuBTyp4Jf6P'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Pablo Oldani', 'userId': 'tx9wovn3Bt7AgDiCB', 'role': {'title': 'Developer', '_id': 'vT85KxNuBTyp4Jf6P'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Sam Beam', 'userId': 'YEFRoDoqsKpKX48J2', 'role': {'title': 'Developer', '_id': 'vT85KxNuBTyp4Jf6P'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Zan Rosenthal', 'userId': 'Funz26f4TWygXPe3N', 'role': {'title': 'Developer', '_id': 'vT85KxNuBTyp4Jf6P'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Byron Matto', 'userId': 'z8J27d3TWsJrYTHKy', 'role': {'title': 'Developer', '_id': 'vT85KxNuBTyp4Jf6P'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Melissa Patterson', 'userId': 'qHqXKZ7kTMrpRZv54', 'role': {'title': 'Developer', '_id': 'vT85KxNuBTyp4Jf6P'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Phil', 'userId': 'kWtmktznBCdLFAdW8', 'role': {'title': 'Developer', '_id': 'vT85KxNuBTyp4Jf6P'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'Stacy Mullins', 'userId': 'tTnxrPytbiaerDT4M', 'role': {'title': 'Developer', '_id': 'vT85KxNuBTyp4Jf6P'}, 'isAdmin': False, 'isPending': False}, {'fullName': 'aislinn@thinkcerca.com', 'userId': '92Y7kEDJfej9xdats', 'role': {'_id': '55555555555555555', 'title': 'Pending'}, 'isAdmin': False, 'isPending': True}]   \n",
       "\n",
       "   active      activePairing  \n",
       "0  True    NaN                \n",
       "1  True    SpiKfuqCoEZRLfDNK  \n",
       "2  True    qhZxmF7byXJszAQ7w  \n",
       "3  True    NaN                \n",
       "4  True    F73t2pJjaXSadWkv8  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "groups = pd.DataFrame(list(db.groups.find({})))\n",
    "\n",
    "# remove testing groups\n",
    "group_creator_ignore_list = ['Demo Admin', 'ykykykykykykykykykyk', 'Stella', 'Kevin Northwestern',\n",
    "                             'Kevin Chen', 'Leesha', 'Jennie']\n",
    "group_id_ignore_list = ['xwDA4HBxXudxF9Swp', '5QXWCwAFBrdbLYGar']\n",
    "\n",
    "group_ignore_ids = list(groups[groups['creatorName'].isin(group_creator_ignore_list)]['_id'].unique()) + group_id_ignore_list\n",
    "\n",
    "# subset groups by id\n",
    "groups_orig_size = len(groups)\n",
    "groups_new_size = 0\n",
    "\n",
    "groups = groups[~groups['_id'].isin(group_ignore_ids)]\n",
    "groups.reset_index(drop=True, inplace=True)\n",
    "\n",
    "# get valid group ids from remaining groups\n",
    "valid_group_ids = groups['_id'].unique()\n",
    "\n",
    "# print change in size\n",
    "groups_new_size = len(groups)\n",
    "print('Number of Groups \\nOriginal size: {} --> New size: {}'.format(groups_orig_size, groups_new_size))\n",
    "\n",
    "# display task history\n",
    "groups.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Users \n",
      "Original size: 1501 --> New size: 1322\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_id</th>\n",
       "      <th>createdAt</th>\n",
       "      <th>services</th>\n",
       "      <th>emails</th>\n",
       "      <th>profile</th>\n",
       "      <th>groups</th>\n",
       "      <th>valid_user</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>22v2zbc6jAjFSwD9s</td>\n",
       "      <td>2020-06-18 00:32:51.415</td>\n",
       "      <td>{'password': {'bcrypt': '$2b$10$Fpjz.um3Jo.7e1nfPoJqyu26YGXjJo8XuqHNRntZve1Z4Wado5pTi'}, 'resume': {'loginTokens': []}}</td>\n",
       "      <td>[{'address': 'NancyGindo2020@u.northwestern.edu', 'verified': True}]</td>\n",
       "      <td>{'fullName': 'Nancy Gindo', 'avatar': 'https://dl.dropboxusercontent.com/s/5ybsquz1ypcku42/profile.png?dl=0'}</td>\n",
       "      <td>[{'groupId': 'nbRvA9o8xBtBNBmsf', 'role': {'title': 'Graduate Student', '_id': 'eGDRZMCTnXjMEYzXG'}, 'groupName': 'AY20 MSC 538-0', 'isAdmin': False, 'isPending': False}]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>27nKbLPrdu2eaHP43</td>\n",
       "      <td>2017-06-17 02:46:55.011</td>\n",
       "      <td>{'password': {'bcrypt': '$2a$10$mrX2Gl//Gehu5jeW.bwIXe0w7yLZPj5BOVvm0NQwj.F.8FixBCfp6'}, 'resume': {'loginTokens': []}}</td>\n",
       "      <td>[{'address': 'michael.jr@northwestern.edu', 'verified': True}]</td>\n",
       "      <td>{'fullName': 'Michael Dice', 'avatar': 'http://orig12.deviantart.net/8670/f/2016/152/b/6/placeholder_1_by_sketchymouse-da4nvhb.png'}</td>\n",
       "      <td>[{'groupId': 'PeACz8rabRn9BJRJc', 'role': {'title': 'Graduate Student', '_id': 'jqfRf62FWQGgjbmBk'}, 'groupName': 'MSC 538-0', 'isAdmin': False, 'isPending': False}, {'groupId': 'fduEdDA8nk5ybcYze', 'role': {'title': 'Staff', '_id': 'CM4sPkMXXxEabRQYL'}, 'groupName': 'fss', 'isAdmin': True, 'isPending': False}, {'groupId': 'nzQTikF493yw5f2jz', 'role': {'title': 'Graduate Student', '_id': 'cQR4rWdCHzczSpRkw'}, 'groupName': 'ETTF Demo', 'isAdmin': False, 'isPending': False}]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2AnxYtnfhpFBdbpEX</td>\n",
       "      <td>2019-04-12 17:06:57.986</td>\n",
       "      <td>{'password': {'bcrypt': '$2a$10$uaUpgdHuD81dJB2jh0daauYZzTMtreHeDEZ1kPMothsCjieN/94Om'}, 'resume': {'loginTokens': []}}</td>\n",
       "      <td>[{'address': 'carylshenry@gmail.com', 'verified': False}]</td>\n",
       "      <td>{'fullName': 'Caryl Henry'}</td>\n",
       "      <td>[{'groupId': 'sM3z5FkZfsABqcj3g', 'role': {'title': 'Undergraduate Student', '_id': 'CwPnwxzSGe2NyRjzx'}, 'groupName': 'Design, Technology, and Research', 'isAdmin': False, 'isPending': False}]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2B7nnQme2DS7kGWkE</td>\n",
       "      <td>2017-08-06 16:12:08.124</td>\n",
       "      <td>{'password': {'bcrypt': '$2a$10$VKfe3nyUFxmXNgDFWN.CpuU1B1U0W9WXKO/lV0lGZId4SMHuNtUSK'}, 'resume': {'loginTokens': []}}</td>\n",
       "      <td>[{'address': 'jgwamuri@gmail.com', 'verified': True}]</td>\n",
       "      <td>{'fullName': 'Jeff Gwamuri', 'avatar': 'http://orig12.deviantart.net/8670/f/2016/152/b/6/placeholder_1_by_sketchymouse-da4nvhb.png'}</td>\n",
       "      <td>[{'groupId': 'dKueghqGY76HqcTqZ', 'role': {'title': 'Member', '_id': 'mFMrA72JE5P6bAee4'}, 'groupName': 'INAT', 'isAdmin': False, 'isPending': False}]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2EmwN2mApJaPrN6sX</td>\n",
       "      <td>2018-05-02 15:34:41.942</td>\n",
       "      <td>{'password': {}}</td>\n",
       "      <td>[{'address': 'NoahConley2015@U.NORTHWESTERN.EDU', 'verified': False}]</td>\n",
       "      <td>{'fullName': 'NoahConley2015@U.NORTHWESTERN.EDU'}</td>\n",
       "      <td>[{'groupId': 'TjwobnBm8xX9BTipw', 'role': {'_id': '55555555555555555', 'title': 'Pending'}, 'groupName': 'CogSci Fest', 'isAdmin': False, 'isPending': True}]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 _id               createdAt  \\\n",
       "1  22v2zbc6jAjFSwD9s 2020-06-18 00:32:51.415   \n",
       "2  27nKbLPrdu2eaHP43 2017-06-17 02:46:55.011   \n",
       "3  2AnxYtnfhpFBdbpEX 2019-04-12 17:06:57.986   \n",
       "4  2B7nnQme2DS7kGWkE 2017-08-06 16:12:08.124   \n",
       "6  2EmwN2mApJaPrN6sX 2018-05-02 15:34:41.942   \n",
       "\n",
       "                                                                                                                  services  \\\n",
       "1  {'password': {'bcrypt': '$2b$10$Fpjz.um3Jo.7e1nfPoJqyu26YGXjJo8XuqHNRntZve1Z4Wado5pTi'}, 'resume': {'loginTokens': []}}   \n",
       "2  {'password': {'bcrypt': '$2a$10$mrX2Gl//Gehu5jeW.bwIXe0w7yLZPj5BOVvm0NQwj.F.8FixBCfp6'}, 'resume': {'loginTokens': []}}   \n",
       "3  {'password': {'bcrypt': '$2a$10$uaUpgdHuD81dJB2jh0daauYZzTMtreHeDEZ1kPMothsCjieN/94Om'}, 'resume': {'loginTokens': []}}   \n",
       "4  {'password': {'bcrypt': '$2a$10$VKfe3nyUFxmXNgDFWN.CpuU1B1U0W9WXKO/lV0lGZId4SMHuNtUSK'}, 'resume': {'loginTokens': []}}   \n",
       "6  {'password': {}}                                                                                                          \n",
       "\n",
       "                                                                  emails  \\\n",
       "1  [{'address': 'NancyGindo2020@u.northwestern.edu', 'verified': True}]    \n",
       "2  [{'address': 'michael.jr@northwestern.edu', 'verified': True}]          \n",
       "3  [{'address': 'carylshenry@gmail.com', 'verified': False}]               \n",
       "4  [{'address': 'jgwamuri@gmail.com', 'verified': True}]                   \n",
       "6  [{'address': 'NoahConley2015@U.NORTHWESTERN.EDU', 'verified': False}]   \n",
       "\n",
       "                                                                                                                                profile  \\\n",
       "1  {'fullName': 'Nancy Gindo', 'avatar': 'https://dl.dropboxusercontent.com/s/5ybsquz1ypcku42/profile.png?dl=0'}                          \n",
       "2  {'fullName': 'Michael Dice', 'avatar': 'http://orig12.deviantart.net/8670/f/2016/152/b/6/placeholder_1_by_sketchymouse-da4nvhb.png'}   \n",
       "3  {'fullName': 'Caryl Henry'}                                                                                                            \n",
       "4  {'fullName': 'Jeff Gwamuri', 'avatar': 'http://orig12.deviantart.net/8670/f/2016/152/b/6/placeholder_1_by_sketchymouse-da4nvhb.png'}   \n",
       "6  {'fullName': 'NoahConley2015@U.NORTHWESTERN.EDU'}                                                                                      \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          groups  \\\n",
       "1  [{'groupId': 'nbRvA9o8xBtBNBmsf', 'role': {'title': 'Graduate Student', '_id': 'eGDRZMCTnXjMEYzXG'}, 'groupName': 'AY20 MSC 538-0', 'isAdmin': False, 'isPending': False}]                                                                                                                                                                                                                                                                                                                      \n",
       "2  [{'groupId': 'PeACz8rabRn9BJRJc', 'role': {'title': 'Graduate Student', '_id': 'jqfRf62FWQGgjbmBk'}, 'groupName': 'MSC 538-0', 'isAdmin': False, 'isPending': False}, {'groupId': 'fduEdDA8nk5ybcYze', 'role': {'title': 'Staff', '_id': 'CM4sPkMXXxEabRQYL'}, 'groupName': 'fss', 'isAdmin': True, 'isPending': False}, {'groupId': 'nzQTikF493yw5f2jz', 'role': {'title': 'Graduate Student', '_id': 'cQR4rWdCHzczSpRkw'}, 'groupName': 'ETTF Demo', 'isAdmin': False, 'isPending': False}]   \n",
       "3  [{'groupId': 'sM3z5FkZfsABqcj3g', 'role': {'title': 'Undergraduate Student', '_id': 'CwPnwxzSGe2NyRjzx'}, 'groupName': 'Design, Technology, and Research', 'isAdmin': False, 'isPending': False}]                                                                                                                                                                                                                                                                                               \n",
       "4  [{'groupId': 'dKueghqGY76HqcTqZ', 'role': {'title': 'Member', '_id': 'mFMrA72JE5P6bAee4'}, 'groupName': 'INAT', 'isAdmin': False, 'isPending': False}]                                                                                                                                                                                                                                                                                                                                          \n",
       "6  [{'groupId': 'TjwobnBm8xX9BTipw', 'role': {'_id': '55555555555555555', 'title': 'Pending'}, 'groupName': 'CogSci Fest', 'isAdmin': False, 'isPending': True}]                                                                                                                                                                                                                                                                                                                                   \n",
       "\n",
       "   valid_user  \n",
       "1  True        \n",
       "2  True        \n",
       "3  True        \n",
       "4  True        \n",
       "6  True        "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "users = pd.DataFrame(list(db.users.find({})))\n",
    "\n",
    "# users must be in at least one valid group\n",
    "valid_group_ids_set = set(valid_group_ids)\n",
    "users['valid_user'] = users['groups'].apply(lambda x: bool(valid_group_ids_set & set([y['groupId'] for y in x])))\n",
    "\n",
    "# remove invalid users\n",
    "users_orig_size = len(users)\n",
    "users_new_size = 0\n",
    "\n",
    "users = users[users['valid_user']]\n",
    "\n",
    "# print change in size\n",
    "users_new_size = len(users)\n",
    "print('Number of Users \\nOriginal size: {} --> New size: {}'.format(users_orig_size, users_new_size))\n",
    "\n",
    "# display users\n",
    "users.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Tasks\n",
      "Original size: 6175 --> New size: 6138\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_id</th>\n",
       "      <th>name</th>\n",
       "      <th>userId</th>\n",
       "      <th>groupId</th>\n",
       "      <th>task</th>\n",
       "      <th>pairingId</th>\n",
       "      <th>group_pairing_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>22JsoKrzhy2eXkse8</td>\n",
       "      <td>Haoqi Zhang</td>\n",
       "      <td>PavTL8zD9664wvtfB</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF</td>\n",
       "      <td>work on my talk</td>\n",
       "      <td>kKja8oadvRjY2xvLr</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-kKja8oadvRjY2xvLr</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>22zPuDWRhaft3TLnX</td>\n",
       "      <td>Alex Kaldjian</td>\n",
       "      <td>aupdNzYu8WmNEi4e5</td>\n",
       "      <td>sM3z5FkZfsABqcj3g</td>\n",
       "      <td>integrating dart script with meteor app</td>\n",
       "      <td>Xkpb46Zc4x3NR8whR</td>\n",
       "      <td>sM3z5FkZfsABqcj3g-Xkpb46Zc4x3NR8whR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>23NBmRrQCWcyLWtFE</td>\n",
       "      <td>Kristine Lu</td>\n",
       "      <td>xQ4mPiD4TX9MJqiqj</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF</td>\n",
       "      <td>Hold me accountable for writing another iteration of dissertation problem and solution statement. (If time, let me talk you through it.)</td>\n",
       "      <td>QLv7Kd3NFjstGYQ25</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-QLv7Kd3NFjstGYQ25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>24HHSxwLxynMa86qb</td>\n",
       "      <td>Nell O'Rourke</td>\n",
       "      <td>CQhpnQnrAQMZhD9tM</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF</td>\n",
       "      <td>Help me work on a template for CS+LS CHI papers (either give me feedback or work quietly)</td>\n",
       "      <td>Qga4xH5NX9bEg3Czm</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-Qga4xH5NX9bEg3Czm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>24TMHPBAX3E9Efor2</td>\n",
       "      <td>Garrett</td>\n",
       "      <td>zmwK4tJHtwLw8pLRC</td>\n",
       "      <td>sM3z5FkZfsABqcj3g</td>\n",
       "      <td>I need to explain something complex to someone and see if it makes sense/ brainstorm ideas on how we can make it make more sense (does this make sense?)</td>\n",
       "      <td>Xie6PLjGLGia7ogYg</td>\n",
       "      <td>sM3z5FkZfsABqcj3g-Xie6PLjGLGia7ogYg</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 _id           name             userId            groupId  \\\n",
       "0  22JsoKrzhy2eXkse8  Haoqi Zhang    PavTL8zD9664wvtfB  9mdkMmj4pY8Q2TwqF   \n",
       "1  22zPuDWRhaft3TLnX  Alex Kaldjian  aupdNzYu8WmNEi4e5  sM3z5FkZfsABqcj3g   \n",
       "2  23NBmRrQCWcyLWtFE  Kristine Lu    xQ4mPiD4TX9MJqiqj  9mdkMmj4pY8Q2TwqF   \n",
       "3  24HHSxwLxynMa86qb  Nell O'Rourke  CQhpnQnrAQMZhD9tM  9mdkMmj4pY8Q2TwqF   \n",
       "4  24TMHPBAX3E9Efor2  Garrett        zmwK4tJHtwLw8pLRC  sM3z5FkZfsABqcj3g   \n",
       "\n",
       "                                                                                                                                                       task  \\\n",
       "0  work on my talk                                                                                                                                            \n",
       "1  integrating dart script with meteor app                                                                                                                    \n",
       "2  Hold me accountable for writing another iteration of dissertation problem and solution statement. (If time, let me talk you through it.)                   \n",
       "3  Help me work on a template for CS+LS CHI papers (either give me feedback or work quietly)                                                                  \n",
       "4  I need to explain something complex to someone and see if it makes sense/ brainstorm ideas on how we can make it make more sense (does this make sense?)   \n",
       "\n",
       "           pairingId                     group_pairing_id  \n",
       "0  kKja8oadvRjY2xvLr  9mdkMmj4pY8Q2TwqF-kKja8oadvRjY2xvLr  \n",
       "1  Xkpb46Zc4x3NR8whR  sM3z5FkZfsABqcj3g-Xkpb46Zc4x3NR8whR  \n",
       "2  QLv7Kd3NFjstGYQ25  9mdkMmj4pY8Q2TwqF-QLv7Kd3NFjstGYQ25  \n",
       "3  Qga4xH5NX9bEg3Czm  9mdkMmj4pY8Q2TwqF-Qga4xH5NX9bEg3Czm  \n",
       "4  Xie6PLjGLGia7ogYg  sM3z5FkZfsABqcj3g-Xie6PLjGLGia7ogYg  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tasks_history = pd.DataFrame(list(db.tasks_history.find({})))\n",
    "\n",
    "# remove bad groups\n",
    "tasks_history_orig_size = len(tasks_history)\n",
    "tasks_history_new_size = 0\n",
    "\n",
    "tasks_history = tasks_history[tasks_history['groupId'].isin(valid_group_ids)]\n",
    "tasks_history.reset_index(drop=True, inplace=True)\n",
    "\n",
    "# add group_pairing_id\n",
    "tasks_history['group_pairing_id'] = tasks_history['groupId'] + '-' + tasks_history['pairingId']\n",
    "\n",
    "# print change in size\n",
    "tasks_history_new_size = len(tasks_history)\n",
    "print('Number of Tasks\\nOriginal size: {} --> New size: {}'.format(tasks_history_orig_size, tasks_history_new_size))\n",
    "\n",
    "# display task history\n",
    "tasks_history.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Pairing Sessions\n",
      "Original size: 894 --> New size: 800\n",
      "Pairing count: 800, Unique group count: 75\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_id</th>\n",
       "      <th>groupId</th>\n",
       "      <th>pairings</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>group_pairing_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>799</th>\n",
       "      <td>Tahmg36pJNJ93m3AW</td>\n",
       "      <td>sM3z5FkZfsABqcj3g</td>\n",
       "      <td>[{'firstUserId': 'GX7okFFb7z4KBHq5E', 'firstUserName': 'Ariella Silver', 'secondUserId': 'bsN2362i2LSv7L2J8', 'secondUserName': 'Jason Friedman'}, {'firstUserId': 'DqXgk8skAodPwr8X6', 'firstUserName': 'Jenny Chang', 'secondUserId': 'mxDCeFFW75vWtXtG6', 'secondUserName': 'Cindy Hu'}, {'firstUserId': 'XnkfNPg3r7oyu6tT4', 'firstUserName': 'Hang Yin', 'secondUserId': 'Z68eenXDt84czPkmN', 'secondUserName': 'Ryan Louie'}, {'firstUserId': 'LXTzT8KpQBpyQdXBD', 'firstUserName': 'Gobi Dasu', 'secondUserId': 'yqoSWNsyy85Zk5p7Q', 'secondUserName': 'jonathanliu2022.1@u.northwestern.edu'}, {'firstUserId': 'uXSSDxWNzRfJDZYYk', 'firstUserName': 'Harrison Kwik', 'secondUserId': 'cQTWHZ5viEh2fns2P', 'secondUserName': 'Molly Pribble'}, {'firstUserId': 'kMpCXyeCS3Sgii22T', 'firstUserName': 'Fardeem Munir'}]</td>\n",
       "      <td>2021-12-03 18:22:12.680</td>\n",
       "      <td>sM3z5FkZfsABqcj3g-Tahmg36pJNJ93m3AW</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>798</th>\n",
       "      <td>EMsEx6rh7ApgDhXf4</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF</td>\n",
       "      <td>[{'firstUserId': 'CftYuz6ZsJ6H92W4m', 'firstUserName': 'Shubhanshi Gaudani', 'secondUserId': 'xQ4mPiD4TX9MJqiqj', 'secondUserName': 'Kristine Lu'}, {'firstUserId': 'LXTzT8KpQBpyQdXBD', 'firstUserName': 'Gobi Dasu', 'secondUserId': 'uXSSDxWNzRfJDZYYk', 'secondUserName': 'Harrison Kwik'}, {'firstUserId': '7yemYfg5RHGKux8YH', 'firstUserName': 'Nick LaGrassa', 'secondUserId': 'qfNF3GukcgyG2dspM', 'secondUserName': 'Evey Huang'}, {'firstUserId': 'Z68eenXDt84czPkmN', 'firstUserName': 'Ryan Louie', 'secondUserId': 'zmwK4tJHtwLw8pLRC', 'secondUserName': 'Garrett'}, {'firstUserId': 'wb8oCHA6zhKkdwfhZ', 'firstUserName': 'Morgan Wu'}]</td>\n",
       "      <td>2021-11-30 20:06:36.983</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-EMsEx6rh7ApgDhXf4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>797</th>\n",
       "      <td>ou82PHwP8gob2s6TA</td>\n",
       "      <td>sM3z5FkZfsABqcj3g</td>\n",
       "      <td>[{'firstUserId': 'kMpCXyeCS3Sgii22T', 'firstUserName': 'Fardeem Munir', 'secondUserId': 'ADqeTpsfbkYDfjZaS', 'secondUserName': 'Roxy Wilcox'}, {'firstUserId': 'mxDCeFFW75vWtXtG6', 'firstUserName': 'Cindy Hu', 'secondUserId': 'DqXgk8skAodPwr8X6', 'secondUserName': 'Jenny Chang'}, {'firstUserId': 'MkwG2fXsuKjRNKWxv', 'firstUserName': 'nataliebrewster2023@u.northwestern.edu', 'secondUserId': '7vKZDX3yH9yojxRiW', 'secondUserName': 'izzychun2022@u.northwestern.edu'}, {'firstUserId': 'LXTzT8KpQBpyQdXBD', 'firstUserName': 'Gobi Dasu', 'secondUserId': 'cQTWHZ5viEh2fns2P', 'secondUserName': 'Molly Pribble'}, {'firstUserId': 'MboYGfwjsk5FFvgsA', 'firstUserName': 'Neha Sharma', 'secondUserId': 'uXSSDxWNzRfJDZYYk', 'secondUserName': 'Harrison Kwik'}, {'firstUserId': 'XnkfNPg3r7oyu6tT4', 'firstUserName': 'Hang Yin', 'secondUserId': 'GX7okFFb7z4KBHq5E', 'secondUserName': 'Ariella Silver'}, {'firstUserId': 'Z68eenXDt84czPkmN', 'firstUserName': 'Ryan Louie', 'secondUserId': 'bsN2362i2LSv7L2J8', 'secondUserName': 'Jason Friedman'}, {'firstUserId': 'yqoSWNsyy85Zk5p7Q', 'firstUserName': 'jonathanliu2022.1@u.northwestern.edu'}]</td>\n",
       "      <td>2021-11-19 18:16:52.519</td>\n",
       "      <td>sM3z5FkZfsABqcj3g-ou82PHwP8gob2s6TA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>796</th>\n",
       "      <td>Q9AvGCemuCNeRfeFY</td>\n",
       "      <td>eKPuSBq5XSLZmMy4g</td>\n",
       "      <td>[{'firstUserId': '76raCBMRqFrNx5pgC', 'firstUserName': 'Geoffrey Litt', 'secondUserId': 'n8M3matQNabdpDZ4N', 'secondUserName': 'Josh Pollock'}, {'firstUserId': 'WThFpaLmD7n6oEkXX', 'firstUserName': 'Stacia Johanna', 'secondUserId': 'YmyfGsFiAvgGAYDv2', 'secondUserName': 'Hope Dargan'}, {'firstUserId': 'ausTGuuD6RdsS4gMG', 'firstUserName': 'Daniel Jackson', 'secondUserId': 'xCnLbAobcKwPq7RD5', 'secondUserName': 'Rob Miller'}]</td>\n",
       "      <td>2021-11-18 16:14:14.616</td>\n",
       "      <td>eKPuSBq5XSLZmMy4g-Q9AvGCemuCNeRfeFY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>795</th>\n",
       "      <td>LgcbkJuSck9Z3AcHJ</td>\n",
       "      <td>erZrXoim85EHwL5Ay</td>\n",
       "      <td>[{'firstUserId': 'HrixEFtNKAozPqu9M', 'firstUserName': 'Elena Glassman', 'secondUserId': 'EQbiHXhYC4xDqiRhe', 'secondUserName': 'tylerholloway@g.harvard.edu'}, {'firstUserId': 'rapxAPeaeKbj2JaRZ', 'firstUserName': 'Litao Yan', 'secondUserId': 'smLoqHiyLYFtQfk4u', 'secondUserName': 'pvaithilingam@g.harvard.edu'}]</td>\n",
       "      <td>2021-11-16 22:09:12.407</td>\n",
       "      <td>erZrXoim85EHwL5Ay-LgcbkJuSck9Z3AcHJ</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   _id            groupId  \\\n",
       "799  Tahmg36pJNJ93m3AW  sM3z5FkZfsABqcj3g   \n",
       "798  EMsEx6rh7ApgDhXf4  9mdkMmj4pY8Q2TwqF   \n",
       "797  ou82PHwP8gob2s6TA  sM3z5FkZfsABqcj3g   \n",
       "796  Q9AvGCemuCNeRfeFY  eKPuSBq5XSLZmMy4g   \n",
       "795  LgcbkJuSck9Z3AcHJ  erZrXoim85EHwL5Ay   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  pairings  \\\n",
       "799  [{'firstUserId': 'GX7okFFb7z4KBHq5E', 'firstUserName': 'Ariella Silver', 'secondUserId': 'bsN2362i2LSv7L2J8', 'secondUserName': 'Jason Friedman'}, {'firstUserId': 'DqXgk8skAodPwr8X6', 'firstUserName': 'Jenny Chang', 'secondUserId': 'mxDCeFFW75vWtXtG6', 'secondUserName': 'Cindy Hu'}, {'firstUserId': 'XnkfNPg3r7oyu6tT4', 'firstUserName': 'Hang Yin', 'secondUserId': 'Z68eenXDt84czPkmN', 'secondUserName': 'Ryan Louie'}, {'firstUserId': 'LXTzT8KpQBpyQdXBD', 'firstUserName': 'Gobi Dasu', 'secondUserId': 'yqoSWNsyy85Zk5p7Q', 'secondUserName': 'jonathanliu2022.1@u.northwestern.edu'}, {'firstUserId': 'uXSSDxWNzRfJDZYYk', 'firstUserName': 'Harrison Kwik', 'secondUserId': 'cQTWHZ5viEh2fns2P', 'secondUserName': 'Molly Pribble'}, {'firstUserId': 'kMpCXyeCS3Sgii22T', 'firstUserName': 'Fardeem Munir'}]                                                                                                                                                                                                                                                                                                                                          \n",
       "798  [{'firstUserId': 'CftYuz6ZsJ6H92W4m', 'firstUserName': 'Shubhanshi Gaudani', 'secondUserId': 'xQ4mPiD4TX9MJqiqj', 'secondUserName': 'Kristine Lu'}, {'firstUserId': 'LXTzT8KpQBpyQdXBD', 'firstUserName': 'Gobi Dasu', 'secondUserId': 'uXSSDxWNzRfJDZYYk', 'secondUserName': 'Harrison Kwik'}, {'firstUserId': '7yemYfg5RHGKux8YH', 'firstUserName': 'Nick LaGrassa', 'secondUserId': 'qfNF3GukcgyG2dspM', 'secondUserName': 'Evey Huang'}, {'firstUserId': 'Z68eenXDt84czPkmN', 'firstUserName': 'Ryan Louie', 'secondUserId': 'zmwK4tJHtwLw8pLRC', 'secondUserName': 'Garrett'}, {'firstUserId': 'wb8oCHA6zhKkdwfhZ', 'firstUserName': 'Morgan Wu'}]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 \n",
       "797  [{'firstUserId': 'kMpCXyeCS3Sgii22T', 'firstUserName': 'Fardeem Munir', 'secondUserId': 'ADqeTpsfbkYDfjZaS', 'secondUserName': 'Roxy Wilcox'}, {'firstUserId': 'mxDCeFFW75vWtXtG6', 'firstUserName': 'Cindy Hu', 'secondUserId': 'DqXgk8skAodPwr8X6', 'secondUserName': 'Jenny Chang'}, {'firstUserId': 'MkwG2fXsuKjRNKWxv', 'firstUserName': 'nataliebrewster2023@u.northwestern.edu', 'secondUserId': '7vKZDX3yH9yojxRiW', 'secondUserName': 'izzychun2022@u.northwestern.edu'}, {'firstUserId': 'LXTzT8KpQBpyQdXBD', 'firstUserName': 'Gobi Dasu', 'secondUserId': 'cQTWHZ5viEh2fns2P', 'secondUserName': 'Molly Pribble'}, {'firstUserId': 'MboYGfwjsk5FFvgsA', 'firstUserName': 'Neha Sharma', 'secondUserId': 'uXSSDxWNzRfJDZYYk', 'secondUserName': 'Harrison Kwik'}, {'firstUserId': 'XnkfNPg3r7oyu6tT4', 'firstUserName': 'Hang Yin', 'secondUserId': 'GX7okFFb7z4KBHq5E', 'secondUserName': 'Ariella Silver'}, {'firstUserId': 'Z68eenXDt84czPkmN', 'firstUserName': 'Ryan Louie', 'secondUserId': 'bsN2362i2LSv7L2J8', 'secondUserName': 'Jason Friedman'}, {'firstUserId': 'yqoSWNsyy85Zk5p7Q', 'firstUserName': 'jonathanliu2022.1@u.northwestern.edu'}]   \n",
       "796  [{'firstUserId': '76raCBMRqFrNx5pgC', 'firstUserName': 'Geoffrey Litt', 'secondUserId': 'n8M3matQNabdpDZ4N', 'secondUserName': 'Josh Pollock'}, {'firstUserId': 'WThFpaLmD7n6oEkXX', 'firstUserName': 'Stacia Johanna', 'secondUserId': 'YmyfGsFiAvgGAYDv2', 'secondUserName': 'Hope Dargan'}, {'firstUserId': 'ausTGuuD6RdsS4gMG', 'firstUserName': 'Daniel Jackson', 'secondUserId': 'xCnLbAobcKwPq7RD5', 'secondUserName': 'Rob Miller'}]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            \n",
       "795  [{'firstUserId': 'HrixEFtNKAozPqu9M', 'firstUserName': 'Elena Glassman', 'secondUserId': 'EQbiHXhYC4xDqiRhe', 'secondUserName': 'tylerholloway@g.harvard.edu'}, {'firstUserId': 'rapxAPeaeKbj2JaRZ', 'firstUserName': 'Litao Yan', 'secondUserId': 'smLoqHiyLYFtQfk4u', 'secondUserName': 'pvaithilingam@g.harvard.edu'}]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               \n",
       "\n",
       "                  timestamp                     group_pairing_id  \n",
       "799 2021-12-03 18:22:12.680  sM3z5FkZfsABqcj3g-Tahmg36pJNJ93m3AW  \n",
       "798 2021-11-30 20:06:36.983  9mdkMmj4pY8Q2TwqF-EMsEx6rh7ApgDhXf4  \n",
       "797 2021-11-19 18:16:52.519  sM3z5FkZfsABqcj3g-ou82PHwP8gob2s6TA  \n",
       "796 2021-11-18 16:14:14.616  eKPuSBq5XSLZmMy4g-Q9AvGCemuCNeRfeFY  \n",
       "795 2021-11-16 22:09:12.407  erZrXoim85EHwL5Ay-LgcbkJuSck9Z3AcHJ  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pairings = pd.DataFrame(list(db.pairings.find({})))\n",
    "\n",
    "# remove bad groups\n",
    "pairings_orig_size = len(pairings)\n",
    "pairings_new_size = 0\n",
    "\n",
    "pairings = pairings[pairings['groupId'].isin(valid_group_ids)]\n",
    "\n",
    "# add group_pair id\n",
    "pairings['group_pairing_id'] = pairings['groupId'] + '-' + pairings['_id']\n",
    "pairings.reset_index(drop=True, inplace=True)\n",
    "\n",
    "# print change in size\n",
    "pairings_new_size = len(pairings)\n",
    "print('Number of Pairing Sessions\\nOriginal size: {} --> New size: {}'.format(pairings_orig_size, pairings_new_size))\n",
    "\n",
    "# display current pairings\n",
    "print('Pairing count: {}, Unique group count: {}'.format(len(pairings), len(pairings.groupId.unique())))\n",
    "pairings.sort_values('timestamp', ascending=False, inplace=True)\n",
    "pairings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Pairs\n",
      "Original size: 4152 --> New size: 4129\n",
      "Unique group count: 75, Unique pairing count: 800\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_id</th>\n",
       "      <th>groupId</th>\n",
       "      <th>pairingId</th>\n",
       "      <th>firstUserId</th>\n",
       "      <th>firstUserName</th>\n",
       "      <th>firstUserRole</th>\n",
       "      <th>secondUserId</th>\n",
       "      <th>secondUserName</th>\n",
       "      <th>secondUserRole</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>group_pairing_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4128</th>\n",
       "      <td>QRv4qhfsRwvgvP6rm</td>\n",
       "      <td>sM3z5FkZfsABqcj3g</td>\n",
       "      <td>Tahmg36pJNJ93m3AW</td>\n",
       "      <td>kMpCXyeCS3Sgii22T</td>\n",
       "      <td>Fardeem Munir</td>\n",
       "      <td>Undergraduate Student</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021-12-03 18:22:12.680</td>\n",
       "      <td>sM3z5FkZfsABqcj3g-Tahmg36pJNJ93m3AW</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4127</th>\n",
       "      <td>zMqhBD8BA46c9E2wM</td>\n",
       "      <td>sM3z5FkZfsABqcj3g</td>\n",
       "      <td>Tahmg36pJNJ93m3AW</td>\n",
       "      <td>uXSSDxWNzRfJDZYYk</td>\n",
       "      <td>Harrison Kwik</td>\n",
       "      <td>Graduate Student</td>\n",
       "      <td>cQTWHZ5viEh2fns2P</td>\n",
       "      <td>Molly Pribble</td>\n",
       "      <td>Undergraduate Student</td>\n",
       "      <td>2021-12-03 18:22:12.680</td>\n",
       "      <td>sM3z5FkZfsABqcj3g-Tahmg36pJNJ93m3AW</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4126</th>\n",
       "      <td>HWq4Ma8fxB3tRYaW8</td>\n",
       "      <td>sM3z5FkZfsABqcj3g</td>\n",
       "      <td>Tahmg36pJNJ93m3AW</td>\n",
       "      <td>LXTzT8KpQBpyQdXBD</td>\n",
       "      <td>Gobi Dasu</td>\n",
       "      <td>Graduate Student</td>\n",
       "      <td>yqoSWNsyy85Zk5p7Q</td>\n",
       "      <td>jonathanliu2022.1@u.northwestern.edu</td>\n",
       "      <td>Undergraduate Student</td>\n",
       "      <td>2021-12-03 18:22:12.680</td>\n",
       "      <td>sM3z5FkZfsABqcj3g-Tahmg36pJNJ93m3AW</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4125</th>\n",
       "      <td>hyYHToRXGDshzmrB4</td>\n",
       "      <td>sM3z5FkZfsABqcj3g</td>\n",
       "      <td>Tahmg36pJNJ93m3AW</td>\n",
       "      <td>XnkfNPg3r7oyu6tT4</td>\n",
       "      <td>Hang Yin</td>\n",
       "      <td>Undergraduate Student</td>\n",
       "      <td>Z68eenXDt84czPkmN</td>\n",
       "      <td>Ryan Louie</td>\n",
       "      <td>Graduate Student</td>\n",
       "      <td>2021-12-03 18:22:12.680</td>\n",
       "      <td>sM3z5FkZfsABqcj3g-Tahmg36pJNJ93m3AW</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4124</th>\n",
       "      <td>D6ZJD7L3YS3PoDG4Y</td>\n",
       "      <td>sM3z5FkZfsABqcj3g</td>\n",
       "      <td>Tahmg36pJNJ93m3AW</td>\n",
       "      <td>DqXgk8skAodPwr8X6</td>\n",
       "      <td>Jenny Chang</td>\n",
       "      <td>Undergraduate Student</td>\n",
       "      <td>mxDCeFFW75vWtXtG6</td>\n",
       "      <td>Cindy Hu</td>\n",
       "      <td>Undergraduate Student</td>\n",
       "      <td>2021-12-03 18:22:12.680</td>\n",
       "      <td>sM3z5FkZfsABqcj3g-Tahmg36pJNJ93m3AW</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4123</th>\n",
       "      <td>bg6Mr7ZB8GtshNR4n</td>\n",
       "      <td>sM3z5FkZfsABqcj3g</td>\n",
       "      <td>Tahmg36pJNJ93m3AW</td>\n",
       "      <td>GX7okFFb7z4KBHq5E</td>\n",
       "      <td>Ariella Silver</td>\n",
       "      <td>Undergraduate Student</td>\n",
       "      <td>bsN2362i2LSv7L2J8</td>\n",
       "      <td>Jason Friedman</td>\n",
       "      <td>Undergraduate Student</td>\n",
       "      <td>2021-12-03 18:22:12.680</td>\n",
       "      <td>sM3z5FkZfsABqcj3g-Tahmg36pJNJ93m3AW</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4119</th>\n",
       "      <td>LmczXeafB45DD4Lc3</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF</td>\n",
       "      <td>EMsEx6rh7ApgDhXf4</td>\n",
       "      <td>LXTzT8KpQBpyQdXBD</td>\n",
       "      <td>Gobi Dasu</td>\n",
       "      <td>PhD Student</td>\n",
       "      <td>uXSSDxWNzRfJDZYYk</td>\n",
       "      <td>Harrison Kwik</td>\n",
       "      <td>PhD Student</td>\n",
       "      <td>2021-11-30 20:06:36.983</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-EMsEx6rh7ApgDhXf4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4118</th>\n",
       "      <td>WPkbSfuB4WKmnCP5H</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF</td>\n",
       "      <td>EMsEx6rh7ApgDhXf4</td>\n",
       "      <td>CftYuz6ZsJ6H92W4m</td>\n",
       "      <td>Shubhanshi Gaudani</td>\n",
       "      <td>Undergraduate Students</td>\n",
       "      <td>xQ4mPiD4TX9MJqiqj</td>\n",
       "      <td>Kristine Lu</td>\n",
       "      <td>PhD Student</td>\n",
       "      <td>2021-11-30 20:06:36.983</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-EMsEx6rh7ApgDhXf4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4122</th>\n",
       "      <td>CZ4FTDwXt4Th6S2Pk</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF</td>\n",
       "      <td>EMsEx6rh7ApgDhXf4</td>\n",
       "      <td>wb8oCHA6zhKkdwfhZ</td>\n",
       "      <td>Morgan Wu</td>\n",
       "      <td>PhD Student</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021-11-30 20:06:36.983</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-EMsEx6rh7ApgDhXf4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4121</th>\n",
       "      <td>TjMPzHFHSx6LZS58Z</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF</td>\n",
       "      <td>EMsEx6rh7ApgDhXf4</td>\n",
       "      <td>Z68eenXDt84czPkmN</td>\n",
       "      <td>Ryan Louie</td>\n",
       "      <td>PhD Student</td>\n",
       "      <td>zmwK4tJHtwLw8pLRC</td>\n",
       "      <td>Garrett</td>\n",
       "      <td>PhD Student</td>\n",
       "      <td>2021-11-30 20:06:36.983</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-EMsEx6rh7ApgDhXf4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    _id            groupId          pairingId  \\\n",
       "4128  QRv4qhfsRwvgvP6rm  sM3z5FkZfsABqcj3g  Tahmg36pJNJ93m3AW   \n",
       "4127  zMqhBD8BA46c9E2wM  sM3z5FkZfsABqcj3g  Tahmg36pJNJ93m3AW   \n",
       "4126  HWq4Ma8fxB3tRYaW8  sM3z5FkZfsABqcj3g  Tahmg36pJNJ93m3AW   \n",
       "4125  hyYHToRXGDshzmrB4  sM3z5FkZfsABqcj3g  Tahmg36pJNJ93m3AW   \n",
       "4124  D6ZJD7L3YS3PoDG4Y  sM3z5FkZfsABqcj3g  Tahmg36pJNJ93m3AW   \n",
       "4123  bg6Mr7ZB8GtshNR4n  sM3z5FkZfsABqcj3g  Tahmg36pJNJ93m3AW   \n",
       "4119  LmczXeafB45DD4Lc3  9mdkMmj4pY8Q2TwqF  EMsEx6rh7ApgDhXf4   \n",
       "4118  WPkbSfuB4WKmnCP5H  9mdkMmj4pY8Q2TwqF  EMsEx6rh7ApgDhXf4   \n",
       "4122  CZ4FTDwXt4Th6S2Pk  9mdkMmj4pY8Q2TwqF  EMsEx6rh7ApgDhXf4   \n",
       "4121  TjMPzHFHSx6LZS58Z  9mdkMmj4pY8Q2TwqF  EMsEx6rh7ApgDhXf4   \n",
       "\n",
       "            firstUserId       firstUserName           firstUserRole  \\\n",
       "4128  kMpCXyeCS3Sgii22T  Fardeem Munir       Undergraduate Student    \n",
       "4127  uXSSDxWNzRfJDZYYk  Harrison Kwik       Graduate Student         \n",
       "4126  LXTzT8KpQBpyQdXBD  Gobi Dasu           Graduate Student         \n",
       "4125  XnkfNPg3r7oyu6tT4  Hang Yin            Undergraduate Student    \n",
       "4124  DqXgk8skAodPwr8X6  Jenny Chang         Undergraduate Student    \n",
       "4123  GX7okFFb7z4KBHq5E  Ariella Silver      Undergraduate Student    \n",
       "4119  LXTzT8KpQBpyQdXBD  Gobi Dasu           PhD Student              \n",
       "4118  CftYuz6ZsJ6H92W4m  Shubhanshi Gaudani  Undergraduate Students   \n",
       "4122  wb8oCHA6zhKkdwfhZ  Morgan Wu           PhD Student              \n",
       "4121  Z68eenXDt84czPkmN  Ryan Louie          PhD Student              \n",
       "\n",
       "           secondUserId                        secondUserName  \\\n",
       "4128  NaN                NaN                                    \n",
       "4127  cQTWHZ5viEh2fns2P  Molly Pribble                          \n",
       "4126  yqoSWNsyy85Zk5p7Q  jonathanliu2022.1@u.northwestern.edu   \n",
       "4125  Z68eenXDt84czPkmN  Ryan Louie                             \n",
       "4124  mxDCeFFW75vWtXtG6  Cindy Hu                               \n",
       "4123  bsN2362i2LSv7L2J8  Jason Friedman                         \n",
       "4119  uXSSDxWNzRfJDZYYk  Harrison Kwik                          \n",
       "4118  xQ4mPiD4TX9MJqiqj  Kristine Lu                            \n",
       "4122  NaN                NaN                                    \n",
       "4121  zmwK4tJHtwLw8pLRC  Garrett                                \n",
       "\n",
       "             secondUserRole               timestamp  \\\n",
       "4128  NaN                   2021-12-03 18:22:12.680   \n",
       "4127  Undergraduate Student 2021-12-03 18:22:12.680   \n",
       "4126  Undergraduate Student 2021-12-03 18:22:12.680   \n",
       "4125  Graduate Student      2021-12-03 18:22:12.680   \n",
       "4124  Undergraduate Student 2021-12-03 18:22:12.680   \n",
       "4123  Undergraduate Student 2021-12-03 18:22:12.680   \n",
       "4119  PhD Student           2021-11-30 20:06:36.983   \n",
       "4118  PhD Student           2021-11-30 20:06:36.983   \n",
       "4122  NaN                   2021-11-30 20:06:36.983   \n",
       "4121  PhD Student           2021-11-30 20:06:36.983   \n",
       "\n",
       "                         group_pairing_id  \n",
       "4128  sM3z5FkZfsABqcj3g-Tahmg36pJNJ93m3AW  \n",
       "4127  sM3z5FkZfsABqcj3g-Tahmg36pJNJ93m3AW  \n",
       "4126  sM3z5FkZfsABqcj3g-Tahmg36pJNJ93m3AW  \n",
       "4125  sM3z5FkZfsABqcj3g-Tahmg36pJNJ93m3AW  \n",
       "4124  sM3z5FkZfsABqcj3g-Tahmg36pJNJ93m3AW  \n",
       "4123  sM3z5FkZfsABqcj3g-Tahmg36pJNJ93m3AW  \n",
       "4119  9mdkMmj4pY8Q2TwqF-EMsEx6rh7ApgDhXf4  \n",
       "4118  9mdkMmj4pY8Q2TwqF-EMsEx6rh7ApgDhXf4  \n",
       "4122  9mdkMmj4pY8Q2TwqF-EMsEx6rh7ApgDhXf4  \n",
       "4121  9mdkMmj4pY8Q2TwqF-EMsEx6rh7ApgDhXf4  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pairs_history = pd.DataFrame(list(db.pairs_history.find({})))\n",
    "\n",
    "# remove bad groups\n",
    "pairs_history_orig_size = len(pairs_history)\n",
    "pairs_history_new_size = 0\n",
    "\n",
    "pairs_history = pairs_history[pairs_history['groupId'].isin(valid_group_ids)]\n",
    "\n",
    "# add group_pairing_id column\n",
    "pairs_history['group_pairing_id'] = pairs_history['groupId'] + '-' + pairs_history['pairingId']\n",
    "pairs_history.reset_index(drop=True, inplace=True)\n",
    "\n",
    "# print change in size\n",
    "pairs_history_new_size = len(pairs_history)\n",
    "print('Number of Pairs\\nOriginal size: {} --> New size: {}'.format(pairs_history_orig_size, pairs_history_new_size))\n",
    "\n",
    "# display current pairs_history\n",
    "print('Unique group count: {}, Unique pairing count: {}'.format(len(pairs_history.groupId.unique()), \n",
    "                                                                len(pairs_history.group_pairing_id.unique())))\n",
    "pairs_history.sort_values('timestamp', ascending=False, inplace=True)\n",
    "pairs_history.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Tasks\n",
      "Original size: 1414 --> New size: 965\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_id</th>\n",
       "      <th>name</th>\n",
       "      <th>userId</th>\n",
       "      <th>groupId</th>\n",
       "      <th>task</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>22ooyXzNXbrGmovfx</td>\n",
       "      <td>JuJu Miao</td>\n",
       "      <td>BNuP3J6tENQwo3s8Z</td>\n",
       "      <td>a5Sj585WvKEA7R5sR</td>\n",
       "      <td>Feedback on general outline of story</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>28XcaSc5mLfpnu6uy</td>\n",
       "      <td>jieun04@gmail.com</td>\n",
       "      <td>PTobofDNubDtM3xTx</td>\n",
       "      <td>PNukLp3iXYBj542kR</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2BLomfERkKHoD3P3G</td>\n",
       "      <td>MedhaviBhasin2015@U.NORTHWESTERN.EDU</td>\n",
       "      <td>6w7vBwqLYi8RbCpR5</td>\n",
       "      <td>TjwobnBm8xX9BTipw</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2CKzLBxMtjx6XvE7h</td>\n",
       "      <td>Justin Krasovetz</td>\n",
       "      <td>yvRvpMKWQZMszeSYv</td>\n",
       "      <td>9FRnvdcgnuX9bYBtJ</td>\n",
       "      <td>Figuring out how to use this application :)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2KAzBjJxnRFME3jaf</td>\n",
       "      <td>maitemarciano2019@u.northwestern.edu</td>\n",
       "      <td>99vdQuxyciRcBohSp</td>\n",
       "      <td>TjwobnBm8xX9BTipw</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 _id                                  name             userId  \\\n",
       "0  22ooyXzNXbrGmovfx  JuJu Miao                             BNuP3J6tENQwo3s8Z   \n",
       "1  28XcaSc5mLfpnu6uy  jieun04@gmail.com                     PTobofDNubDtM3xTx   \n",
       "2  2BLomfERkKHoD3P3G  MedhaviBhasin2015@U.NORTHWESTERN.EDU  6w7vBwqLYi8RbCpR5   \n",
       "3  2CKzLBxMtjx6XvE7h  Justin Krasovetz                      yvRvpMKWQZMszeSYv   \n",
       "4  2KAzBjJxnRFME3jaf  maitemarciano2019@u.northwestern.edu  99vdQuxyciRcBohSp   \n",
       "\n",
       "             groupId                                         task  \n",
       "0  a5Sj585WvKEA7R5sR  Feedback on general outline of story         \n",
       "1  PNukLp3iXYBj542kR  NaN                                          \n",
       "2  TjwobnBm8xX9BTipw  NaN                                          \n",
       "3  9FRnvdcgnuX9bYBtJ  Figuring out how to use this application :)  \n",
       "4  TjwobnBm8xX9BTipw  NaN                                          "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tasks = pd.DataFrame(list(db.tasks.find({})))\n",
    "\n",
    "# remove bad groups\n",
    "tasks_orig_size = len(tasks)\n",
    "tasks_new_size = 0\n",
    "\n",
    "tasks = tasks[tasks['groupId'].isin(valid_group_ids)]\n",
    "tasks.reset_index(drop=True, inplace=True)\n",
    "\n",
    "# print change in size\n",
    "tasks_new_size = len(tasks)\n",
    "print('Number of Tasks\\nOriginal size: {} --> New size: {}'.format(tasks_orig_size, tasks_new_size))\n",
    "\n",
    "# display current tasks\n",
    "tasks.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Current Affinities\n",
      "Original size: 4489 --> New size: 3063\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_id</th>\n",
       "      <th>helperId</th>\n",
       "      <th>helpeeId</th>\n",
       "      <th>groupId</th>\n",
       "      <th>value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>245JFYNd6XNwk5nbd</td>\n",
       "      <td>acoFQ6xXo29zBPcPp</td>\n",
       "      <td>kMEKB25ouoW8nM5JP</td>\n",
       "      <td>u4kjJC55DPMLpR8bC</td>\n",
       "      <td>0.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>25qLZMJ5MWr63YTHP</td>\n",
       "      <td>Tu3L7LpisKfH8YTnp</td>\n",
       "      <td>YhBpcJFgKkrHmSxpg</td>\n",
       "      <td>qyiCF6mPcE2x9rQFa</td>\n",
       "      <td>0.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>29PWAM28tycxNsyAP</td>\n",
       "      <td>5swmuvTkNq9eD44Hb</td>\n",
       "      <td>FL8FPTrpeTGLpjAQE</td>\n",
       "      <td>nbRvA9o8xBtBNBmsf</td>\n",
       "      <td>0.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2BP38eymyv2G2De2h</td>\n",
       "      <td>FAWQBqqQESL6uQBkc</td>\n",
       "      <td>Q9SBbRXjecTYHDemQ</td>\n",
       "      <td>nbRvA9o8xBtBNBmsf</td>\n",
       "      <td>0.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2CNymZxEgF46rWL3A</td>\n",
       "      <td>uQX4GM7PdysSZve4j</td>\n",
       "      <td>kmKQMnmKmgqgNpyNW</td>\n",
       "      <td>CFH2M9XxiETJZjxRR</td>\n",
       "      <td>0.33</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 _id           helperId           helpeeId            groupId  \\\n",
       "0  245JFYNd6XNwk5nbd  acoFQ6xXo29zBPcPp  kMEKB25ouoW8nM5JP  u4kjJC55DPMLpR8bC   \n",
       "1  25qLZMJ5MWr63YTHP  Tu3L7LpisKfH8YTnp  YhBpcJFgKkrHmSxpg  qyiCF6mPcE2x9rQFa   \n",
       "2  29PWAM28tycxNsyAP  5swmuvTkNq9eD44Hb  FL8FPTrpeTGLpjAQE  nbRvA9o8xBtBNBmsf   \n",
       "3  2BP38eymyv2G2De2h  FAWQBqqQESL6uQBkc  Q9SBbRXjecTYHDemQ  nbRvA9o8xBtBNBmsf   \n",
       "4  2CNymZxEgF46rWL3A  uQX4GM7PdysSZve4j  kmKQMnmKmgqgNpyNW  CFH2M9XxiETJZjxRR   \n",
       "\n",
       "   value  \n",
       "0  0.33   \n",
       "1  0.33   \n",
       "2  0.33   \n",
       "3  0.33   \n",
       "4  0.33   "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "affinities = pd.DataFrame(list(db.affinities.find({})))\n",
    "\n",
    "# remove bad groups\n",
    "affinities_orig_size = len(affinities)\n",
    "affinities_new_size = 0\n",
    "\n",
    "affinities = affinities[affinities['groupId'].isin(valid_group_ids)]\n",
    "affinities.reset_index(drop=True, inplace=True)\n",
    "\n",
    "# print change in size\n",
    "affinities_new_size = len(affinities)\n",
    "print('Number of Current Affinities\\nOriginal size: {} --> New size: {}'.format(affinities_orig_size, affinities_new_size))\n",
    "\n",
    "# display current affinities\n",
    "affinities.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Past Affinities\n",
      "Original size: 71943 --> New size: 71581\n",
      "Unique Group Pairings: 777\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_id</th>\n",
       "      <th>helperId</th>\n",
       "      <th>helpeeId</th>\n",
       "      <th>groupId</th>\n",
       "      <th>value</th>\n",
       "      <th>pairingId</th>\n",
       "      <th>group_pairing_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>v3nKkg77Jouf6BZ8G</td>\n",
       "      <td>GLTz7m8y7RqZCYzxx</td>\n",
       "      <td>3si95Pn6NjXTxCWcT</td>\n",
       "      <td>2rFoGTfRa9LFdpQNA</td>\n",
       "      <td>0.33</td>\n",
       "      <td>2EPbA6HkydPTdxCWD</td>\n",
       "      <td>2rFoGTfRa9LFdpQNA-2EPbA6HkydPTdxCWD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>D2kBQDRftmygv5f4L</td>\n",
       "      <td>PWufwHDsbRaw4se4X</td>\n",
       "      <td>3si95Pn6NjXTxCWcT</td>\n",
       "      <td>2rFoGTfRa9LFdpQNA</td>\n",
       "      <td>1.00</td>\n",
       "      <td>2EPbA6HkydPTdxCWD</td>\n",
       "      <td>2rFoGTfRa9LFdpQNA-2EPbA6HkydPTdxCWD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>R588B5nqLhmLbC4iW</td>\n",
       "      <td>f8wwqTXaifkxxoAc2</td>\n",
       "      <td>3si95Pn6NjXTxCWcT</td>\n",
       "      <td>2rFoGTfRa9LFdpQNA</td>\n",
       "      <td>0.00</td>\n",
       "      <td>2EPbA6HkydPTdxCWD</td>\n",
       "      <td>2rFoGTfRa9LFdpQNA-2EPbA6HkydPTdxCWD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>poiynLy2tnCMNzdGf</td>\n",
       "      <td>iyRaCwz7QzxPRSi5t</td>\n",
       "      <td>3si95Pn6NjXTxCWcT</td>\n",
       "      <td>2rFoGTfRa9LFdpQNA</td>\n",
       "      <td>1.00</td>\n",
       "      <td>2EPbA6HkydPTdxCWD</td>\n",
       "      <td>2rFoGTfRa9LFdpQNA-2EPbA6HkydPTdxCWD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>KmiSFQicDRa263Nfc</td>\n",
       "      <td>kEZXdjhfohiGxJWdu</td>\n",
       "      <td>3si95Pn6NjXTxCWcT</td>\n",
       "      <td>2rFoGTfRa9LFdpQNA</td>\n",
       "      <td>-1.00</td>\n",
       "      <td>2EPbA6HkydPTdxCWD</td>\n",
       "      <td>2rFoGTfRa9LFdpQNA-2EPbA6HkydPTdxCWD</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 _id           helperId           helpeeId            groupId  \\\n",
       "0  v3nKkg77Jouf6BZ8G  GLTz7m8y7RqZCYzxx  3si95Pn6NjXTxCWcT  2rFoGTfRa9LFdpQNA   \n",
       "1  D2kBQDRftmygv5f4L  PWufwHDsbRaw4se4X  3si95Pn6NjXTxCWcT  2rFoGTfRa9LFdpQNA   \n",
       "2  R588B5nqLhmLbC4iW  f8wwqTXaifkxxoAc2  3si95Pn6NjXTxCWcT  2rFoGTfRa9LFdpQNA   \n",
       "3  poiynLy2tnCMNzdGf  iyRaCwz7QzxPRSi5t  3si95Pn6NjXTxCWcT  2rFoGTfRa9LFdpQNA   \n",
       "4  KmiSFQicDRa263Nfc  kEZXdjhfohiGxJWdu  3si95Pn6NjXTxCWcT  2rFoGTfRa9LFdpQNA   \n",
       "\n",
       "   value          pairingId                     group_pairing_id  \n",
       "0  0.33   2EPbA6HkydPTdxCWD  2rFoGTfRa9LFdpQNA-2EPbA6HkydPTdxCWD  \n",
       "1  1.00   2EPbA6HkydPTdxCWD  2rFoGTfRa9LFdpQNA-2EPbA6HkydPTdxCWD  \n",
       "2  0.00   2EPbA6HkydPTdxCWD  2rFoGTfRa9LFdpQNA-2EPbA6HkydPTdxCWD  \n",
       "3  1.00   2EPbA6HkydPTdxCWD  2rFoGTfRa9LFdpQNA-2EPbA6HkydPTdxCWD  \n",
       "4 -1.00   2EPbA6HkydPTdxCWD  2rFoGTfRa9LFdpQNA-2EPbA6HkydPTdxCWD  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "affinities_history = pd.DataFrame(list(db.affinities_history.find({})))\n",
    "\n",
    "# remove bad groups\n",
    "affinities_history_orig_size = len(affinities_history)\n",
    "affinities_history_new_size = 0\n",
    "\n",
    "affinities_history = affinities_history[affinities_history['groupId'].isin(valid_group_ids)]\n",
    "\n",
    "# add group_pairing_id column\n",
    "affinities_history['group_pairing_id'] = affinities_history['groupId'] + '-' + affinities_history['pairingId']\n",
    "\n",
    "# remove duplicate ratings\n",
    "affinities_history.sort_values(['group_pairing_id', 'helpeeId', 'helperId'], inplace=True)\n",
    "affinities_history.drop_duplicates(subset=['group_pairing_id', 'helpeeId', 'helperId'], keep='last', inplace=True)\n",
    "affinities_history.reset_index(drop=True, inplace=True)\n",
    "\n",
    "# print change in size\n",
    "affinities_history_new_size = len(affinities_history)\n",
    "print('Number of Past Affinities\\nOriginal size: {} --> New size: {}'.format(affinities_history_orig_size, affinities_history_new_size))\n",
    "\n",
    "# display affinity data\n",
    "print('Unique Group Pairings: {}'.format(len(affinities_history.group_pairing_id.unique())))\n",
    "affinities_history.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cleaning Data\n",
    "Make sure all pairing sessions are valid. A valid pairing session from the `pairings` table for a group-pairing ID will have $n$ members in the 'pairings' column. \n",
    "\n",
    "It must also have:\n",
    "1. $\\frac{n}{2}$ pairs in the 'pairings' column of the `pairings` table for that group-pairing ID entry.\n",
    "2. $\\frac{n}{2}$ rows in the `pairs_history` table for rows with the same group-pairing ID entry.\n",
    "3. $n$ tasks in the `tasks_history` table for rows with the same group-pairing ID entry.\n",
    "\n",
    "Ideally, data should have the following, but these are not guaranteed since users may not report affinities for all users in the current pool.\n",
    "4. $n*(n - 1)$ total affinities in the `affinities_history` table for rows with the same group-pairing ID entry.\n",
    "5. $n - 1$ affinities per person in the `affinities_history` table for rows with the same group-pairing ID entry."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TODO\n",
    "- Unchecked edge case: when pair research doesn't pair everyone given an even number of users\n",
    "- Plot when different kinds of conditions are breaking as a bar plot (aggregate), and a timeseries plot (trend) to show if phenomena is ongoing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Determine invalid group-pairing sessions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_members_in_pairing(pairing):\n",
    "    \"\"\"\n",
    "    Counts number of members in a pairing.\n",
    "    \n",
    "    Input:\n",
    "        pairing (list of dicts): pairings between members as lists of dicts.\n",
    "    \n",
    "    Output:\n",
    "        (int): number of people in the pairing.\n",
    "    \"\"\"\n",
    "    count = 0\n",
    "    for pair in pairing:\n",
    "        if 'firstUserId' in pair and pair['firstUserId'] is not None:\n",
    "            count += 1\n",
    "        if 'secondUserId' in pair and pair['secondUserId'] is not None:\n",
    "            count += 1\n",
    "            \n",
    "    return count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def validate_pairing(group_pairing_id, debug=False):\n",
    "    \"\"\"\n",
    "    Validates that all data for a pairing is good.\n",
    "    \n",
    "    A valid pairing with n particpants for a given group_pairing_id meets the following conditions:\n",
    "    1. n/2 pairs in the 'pairings' column of the `pairings` table for that group_pairing_id entry.\n",
    "    2. n/2 rows in the `pairs_history` table for rows with the same group_pairing_id entry.\n",
    "    3. n tasks in the `tasks_history` table for rows with the same group_pairing_id entry.\n",
    "    \n",
    "    Input:\n",
    "        group_pairing_id (string): pairing session for group to validate.\n",
    "        debug (bool): optional parameter to print whenever invalid session is detected.\n",
    "        \n",
    "    Output:\n",
    "        (bool): whether pairing is valid\n",
    "        (list of string): conditions failed in check\n",
    "    \"\"\"\n",
    "    # get the pairing and number of users\n",
    "    curr_pairing = pairings.query(\"group_pairing_id == @group_pairing_id\").iloc[0]['pairings']\n",
    "    n = count_members_in_pairing(curr_pairing)\n",
    "    pairs_count = math.ceil(n / 2)\n",
    "    \n",
    "    # store each condition check\n",
    "    condition_checks = [False for x in range(3)]\n",
    "    \n",
    "    # check condition 1\n",
    "    condition_checks[0] = len(curr_pairing) == pairs_count\n",
    "    \n",
    "    # check condition 2\n",
    "    condition_checks[1] = len(pairs_history.query(\"group_pairing_id == @group_pairing_id\")) == pairs_count\n",
    "    \n",
    "    # check condition 3\n",
    "    temp_tasks_history = tasks_history.query(\"group_pairing_id == @group_pairing_id\").copy(deep=True)\n",
    "    if len(temp_tasks_history) > 0:\n",
    "        temp_tasks_history['group_pairing_user_id'] = temp_tasks_history.apply(lambda x: '{}-{}-{}'.format(x['groupId'], x['pairingId'], x['userId']), axis=1)\n",
    "        temp_tasks_history.drop_duplicates(['group_pairing_user_id'], keep=False, inplace=True)\n",
    "        condition_checks[2] = len(temp_tasks_history) == n\n",
    "    \n",
    "    # return checks\n",
    "    all_conds_valid = all(condition_checks)\n",
    "    failed_conds = None\n",
    "    \n",
    "    if not all_conds_valid:\n",
    "        failed_conds = [str(index + 1) for index, condition in enumerate(condition_checks) if not condition]\n",
    "        \n",
    "        if debug:\n",
    "            print('Invalid Group-Pairing Session: {} | Check Conditions Failed: {}'.format(group_pairing_id, ', '.join(failed_conds)))\n",
    "\n",
    "    return all_conds_valid, failed_conds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f1f34599368748fda191aa8f1444cda3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/800 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>group_pairing_id</th>\n",
       "      <th>group_id</th>\n",
       "      <th>pairing_id</th>\n",
       "      <th>conditions_failed</th>\n",
       "      <th>user_count</th>\n",
       "      <th>task_count</th>\n",
       "      <th>expected_task_count</th>\n",
       "      <th>pairing_count</th>\n",
       "      <th>expected_pairing_count</th>\n",
       "      <th>pairs_hist_count</th>\n",
       "      <th>expected_pairs_hist_count</th>\n",
       "      <th>affinity_count</th>\n",
       "      <th>expected_affinity_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>PeACz8rabRn9BJRJc-3ksC39hwSs3ZzG9B8</td>\n",
       "      <td>PeACz8rabRn9BJRJc</td>\n",
       "      <td>3ksC39hwSs3ZzG9B8</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>14.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>109.0</td>\n",
       "      <td>182.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Pj9mwxXYPKxjPaxST-mgoxZfxyary28EXFn</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST</td>\n",
       "      <td>mgoxZfxyary28EXFn</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>56.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-P8SGjH7mLk9FxCf45</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF</td>\n",
       "      <td>P8SGjH7mLk9FxCf45</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>12.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>132.0</td>\n",
       "      <td>132.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-WNJjNxKHc5ubX2jwc</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF</td>\n",
       "      <td>WNJjNxKHc5ubX2jwc</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>16.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>234.0</td>\n",
       "      <td>240.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>BibLRuKtNNv7QEDqb-jdfB7xy4iuaG66yiZ</td>\n",
       "      <td>BibLRuKtNNv7QEDqb</td>\n",
       "      <td>jdfB7xy4iuaG66yiZ</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>56.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      group_pairing_id           group_id         pairing_id  \\\n",
       "0  PeACz8rabRn9BJRJc-3ksC39hwSs3ZzG9B8  PeACz8rabRn9BJRJc  3ksC39hwSs3ZzG9B8   \n",
       "1  Pj9mwxXYPKxjPaxST-mgoxZfxyary28EXFn  Pj9mwxXYPKxjPaxST  mgoxZfxyary28EXFn   \n",
       "2  9mdkMmj4pY8Q2TwqF-P8SGjH7mLk9FxCf45  9mdkMmj4pY8Q2TwqF  P8SGjH7mLk9FxCf45   \n",
       "3  9mdkMmj4pY8Q2TwqF-WNJjNxKHc5ubX2jwc  9mdkMmj4pY8Q2TwqF  WNJjNxKHc5ubX2jwc   \n",
       "4  BibLRuKtNNv7QEDqb-jdfB7xy4iuaG66yiZ  BibLRuKtNNv7QEDqb  jdfB7xy4iuaG66yiZ   \n",
       "\n",
       "  conditions_failed  user_count  task_count  expected_task_count  \\\n",
       "0  [1, 2]            14.0        14.0        14.0                  \n",
       "1  [1, 2]            8.0         8.0         8.0                   \n",
       "2  [1, 2]            12.0        12.0        12.0                  \n",
       "3  [1, 2]            16.0        16.0        16.0                  \n",
       "4  [1, 2]            8.0         8.0         8.0                   \n",
       "\n",
       "   pairing_count  expected_pairing_count  pairs_hist_count  \\\n",
       "0  8.0            7.0                     8.0                \n",
       "1  5.0            4.0                     5.0                \n",
       "2  7.0            6.0                     7.0                \n",
       "3  9.0            8.0                     9.0                \n",
       "4  5.0            4.0                     5.0                \n",
       "\n",
       "   expected_pairs_hist_count  affinity_count  expected_affinity_count  \n",
       "0  7.0                        109.0           182.0                    \n",
       "1  4.0                        48.0            56.0                     \n",
       "2  6.0                        132.0           132.0                    \n",
       "3  8.0                        234.0           240.0                    \n",
       "4  4.0                        49.0            56.0                     "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get group\n",
    "group_pairing_ids = pairings['group_pairing_id'].unique()\n",
    "group_pairing_ids.sort()\n",
    "\n",
    "# collect all invalid group-pairing sessions\n",
    "invalid_group_pairings = pd.DataFrame({\n",
    "    'group_pairing_id': [],\n",
    "    'group_id': [],\n",
    "    'pairing_id': [],\n",
    "    'conditions_failed': [],\n",
    "    'user_count': [],\n",
    "    'task_count': [],\n",
    "    'expected_task_count': [],\n",
    "    'pairing_count': [],\n",
    "    'expected_pairing_count': [],\n",
    "    'pairs_hist_count': [],\n",
    "    'expected_pairs_hist_count': [],\n",
    "    'affinity_count': [],\n",
    "    'expected_affinity_count': []\n",
    "})\n",
    "\n",
    "for curr_id in tqdm(group_pairing_ids):\n",
    "    curr_group_id, curr_pairing_id = curr_id.split('-')\n",
    "    is_valid_pairing, conditions_failed = validate_pairing(curr_id)\n",
    "    \n",
    "    if not is_valid_pairing:\n",
    "        n = count_members_in_pairing(pairings.query(\"group_pairing_id == @curr_id\").iloc[0]['pairings'])\n",
    "        \n",
    "        invalid_group_pairings = invalid_group_pairings.append({\n",
    "            'group_pairing_id': curr_id,\n",
    "            'group_id': curr_group_id,\n",
    "            'pairing_id': curr_pairing_id,\n",
    "            'conditions_failed': conditions_failed,\n",
    "            'user_count': n,\n",
    "            'task_count': len(tasks_history.query(\"group_pairing_id == @curr_id\")),\n",
    "            'expected_task_count': n,\n",
    "            'pairing_count': len(pairings.query(\"group_pairing_id == @curr_id\").iloc[0]['pairings']),\n",
    "            'expected_pairing_count': math.ceil(n / 2),\n",
    "            'pairs_hist_count': len(pairs_history.query(\"group_pairing_id == @curr_id\")),\n",
    "            'expected_pairs_hist_count': math.ceil(n / 2),\n",
    "            'affinity_count': len(affinities_history.query(\"group_pairing_id == @curr_id\")),\n",
    "            'expected_affinity_count': n * (n - 1)\n",
    "        }, ignore_index=True)\n",
    "        \n",
    "invalid_group_pairings.sort_values('conditions_failed', inplace=True)\n",
    "invalid_group_pairings.reset_index(drop=True, inplace=True)\n",
    "invalid_group_pairings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>group_pairing_id</th>\n",
       "      <th>group_id</th>\n",
       "      <th>pairing_id</th>\n",
       "      <th>conditions_failed</th>\n",
       "      <th>user_count</th>\n",
       "      <th>task_count</th>\n",
       "      <th>expected_task_count</th>\n",
       "      <th>pairing_count</th>\n",
       "      <th>expected_pairing_count</th>\n",
       "      <th>pairs_hist_count</th>\n",
       "      <th>expected_pairs_hist_count</th>\n",
       "      <th>affinity_count</th>\n",
       "      <th>expected_affinity_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>PeACz8rabRn9BJRJc-3ksC39hwSs3ZzG9B8</td>\n",
       "      <td>PeACz8rabRn9BJRJc</td>\n",
       "      <td>3ksC39hwSs3ZzG9B8</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>14.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>109.0</td>\n",
       "      <td>182.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Pj9mwxXYPKxjPaxST-mgoxZfxyary28EXFn</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST</td>\n",
       "      <td>mgoxZfxyary28EXFn</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>56.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-P8SGjH7mLk9FxCf45</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF</td>\n",
       "      <td>P8SGjH7mLk9FxCf45</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>12.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>132.0</td>\n",
       "      <td>132.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-WNJjNxKHc5ubX2jwc</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF</td>\n",
       "      <td>WNJjNxKHc5ubX2jwc</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>16.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>234.0</td>\n",
       "      <td>240.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>BibLRuKtNNv7QEDqb-jdfB7xy4iuaG66yiZ</td>\n",
       "      <td>BibLRuKtNNv7QEDqb</td>\n",
       "      <td>jdfB7xy4iuaG66yiZ</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>56.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-hL8oByh2FnRJSk7LL</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF</td>\n",
       "      <td>hL8oByh2FnRJSk7LL</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>90.0</td>\n",
       "      <td>90.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-mKv8rBNJhkFbmSarX</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF</td>\n",
       "      <td>mKv8rBNJhkFbmSarX</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>16.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>234.0</td>\n",
       "      <td>240.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-4dEvFqkiADYDtSNeR</td>\n",
       "      <td>sM3z5FkZfsABqcj3g</td>\n",
       "      <td>4dEvFqkiADYDtSNeR</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>82.0</td>\n",
       "      <td>90.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-qcYXZGnrw3cmEsL5v</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF</td>\n",
       "      <td>qcYXZGnrw3cmEsL5v</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>16.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>240.0</td>\n",
       "      <td>240.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>BibLRuKtNNv7QEDqb-r7cB6GdSpMjLpK8Jc</td>\n",
       "      <td>BibLRuKtNNv7QEDqb</td>\n",
       "      <td>r7cB6GdSpMjLpK8Jc</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>90.0</td>\n",
       "      <td>90.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>BibLRuKtNNv7QEDqb-Q9qoS4WQTWAEw47NH</td>\n",
       "      <td>BibLRuKtNNv7QEDqb</td>\n",
       "      <td>Q9qoS4WQTWAEw47NH</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>12.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>BibLRuKtNNv7QEDqb-QewbaRfokgfMFiD5C</td>\n",
       "      <td>BibLRuKtNNv7QEDqb</td>\n",
       "      <td>QewbaRfokgfMFiD5C</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>18.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>306.0</td>\n",
       "      <td>306.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>cToFEbgXcFbrKsSrj-o33KWpqZFwSNt3tDe</td>\n",
       "      <td>cToFEbgXcFbrKsSrj</td>\n",
       "      <td>o33KWpqZFwSNt3tDe</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>69.0</td>\n",
       "      <td>90.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>BibLRuKtNNv7QEDqb-YuHJWnrgg2prjPjRp</td>\n",
       "      <td>BibLRuKtNNv7QEDqb</td>\n",
       "      <td>YuHJWnrgg2prjPjRp</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>12.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>BibLRuKtNNv7QEDqb-evGbShCDMHxvCBABe</td>\n",
       "      <td>BibLRuKtNNv7QEDqb</td>\n",
       "      <td>evGbShCDMHxvCBABe</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>30.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>ckNqEpoM9jbCER9ex-qNMTGWnYiCzfSdRzv</td>\n",
       "      <td>ckNqEpoM9jbCER9ex</td>\n",
       "      <td>qNMTGWnYiCzfSdRzv</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>14.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>182.0</td>\n",
       "      <td>182.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-CJEjiJ2v3Wmq7SXnK</td>\n",
       "      <td>sM3z5FkZfsABqcj3g</td>\n",
       "      <td>CJEjiJ2v3Wmq7SXnK</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>82.0</td>\n",
       "      <td>90.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>BibLRuKtNNv7QEDqb-mMf4jK8LZ9cBY7yJK</td>\n",
       "      <td>BibLRuKtNNv7QEDqb</td>\n",
       "      <td>mMf4jK8LZ9cBY7yJK</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>55.0</td>\n",
       "      <td>56.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>PeACz8rabRn9BJRJc-w8mMv9X4eQe8y7tmG</td>\n",
       "      <td>PeACz8rabRn9BJRJc</td>\n",
       "      <td>w8mMv9X4eQe8y7tmG</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>12.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-jZaAYDh4ZgYESGsT3</td>\n",
       "      <td>sM3z5FkZfsABqcj3g</td>\n",
       "      <td>jZaAYDh4ZgYESGsT3</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>14.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>155.0</td>\n",
       "      <td>182.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-wrkkbAdZf9AMeKgsH</td>\n",
       "      <td>sM3z5FkZfsABqcj3g</td>\n",
       "      <td>wrkkbAdZf9AMeKgsH</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>85.0</td>\n",
       "      <td>90.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Pj9mwxXYPKxjPaxST-MmfKpPtSv5TWDy8Jc</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST</td>\n",
       "      <td>MmfKpPtSv5TWDy8Jc</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>55.0</td>\n",
       "      <td>56.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>cToFEbgXcFbrKsSrj-9LidQoQiuSXdwJwoc</td>\n",
       "      <td>cToFEbgXcFbrKsSrj</td>\n",
       "      <td>9LidQoQiuSXdwJwoc</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>14.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>134.0</td>\n",
       "      <td>182.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>cToFEbgXcFbrKsSrj-Kevyvh5GoYBa6QhtS</td>\n",
       "      <td>cToFEbgXcFbrKsSrj</td>\n",
       "      <td>Kevyvh5GoYBa6QhtS</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>14.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>133.0</td>\n",
       "      <td>182.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt</td>\n",
       "      <td>cToFEbgXcFbrKsSrj</td>\n",
       "      <td>kTtxafCNsKoqsnhCt</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>14.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>131.0</td>\n",
       "      <td>182.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>fduEdDA8nk5ybcYze-9deZN5msXPWbRbaty</td>\n",
       "      <td>fduEdDA8nk5ybcYze</td>\n",
       "      <td>9deZN5msXPWbRbaty</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>fduEdDA8nk5ybcYze-GjXnW5cj2SE8rzNCJ</td>\n",
       "      <td>fduEdDA8nk5ybcYze</td>\n",
       "      <td>GjXnW5cj2SE8rzNCJ</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>fduEdDA8nk5ybcYze-vyrrtwEpzzRqCACxi</td>\n",
       "      <td>fduEdDA8nk5ybcYze</td>\n",
       "      <td>vyrrtwEpzzRqCACxi</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-nRAQpsPhsQs4zRvTL</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF</td>\n",
       "      <td>nRAQpsPhsQs4zRvTL</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>10.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>90.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>tAC6QTAqiEgFRpHMj-7JhveY48LSxc8s87j</td>\n",
       "      <td>tAC6QTAqiEgFRpHMj</td>\n",
       "      <td>7JhveY48LSxc8s87j</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>FmHT4pnqrPXzCLCmE-z8eGJ6ktY8jpBaAXN</td>\n",
       "      <td>FmHT4pnqrPXzCLCmE</td>\n",
       "      <td>z8eGJ6ktY8jpBaAXN</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-Q2Qffh22oZwWrRCWt</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF</td>\n",
       "      <td>Q2Qffh22oZwWrRCWt</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>8.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>56.0</td>\n",
       "      <td>56.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>FmHT4pnqrPXzCLCmE-EbjzaxZ6wADuYQKqq</td>\n",
       "      <td>FmHT4pnqrPXzCLCmE</td>\n",
       "      <td>EbjzaxZ6wADuYQKqq</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>qyiCF6mPcE2x9rQFa-452ZTHfYTnMazmiEg</td>\n",
       "      <td>qyiCF6mPcE2x9rQFa</td>\n",
       "      <td>452ZTHfYTnMazmiEg</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>Pj9mwxXYPKxjPaxST-Tho38bGKWrSDgZbJA</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST</td>\n",
       "      <td>Tho38bGKWrSDgZbJA</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>55.0</td>\n",
       "      <td>56.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>KaYHdgPnyHQx84CPL-cqPJEg45npP4KZZqS</td>\n",
       "      <td>KaYHdgPnyHQx84CPL</td>\n",
       "      <td>cqPJEg45npP4KZZqS</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>8.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>56.0</td>\n",
       "      <td>56.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>Pj9mwxXYPKxjPaxST-ZEgMQhsGM5LM8HTJ7</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST</td>\n",
       "      <td>ZEgMQhsGM5LM8HTJ7</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>55.0</td>\n",
       "      <td>56.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>Pj9mwxXYPKxjPaxST-kDNiwKsweERmTmqqR</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST</td>\n",
       "      <td>kDNiwKsweERmTmqqR</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>55.0</td>\n",
       "      <td>56.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       group_pairing_id           group_id         pairing_id  \\\n",
       "0   PeACz8rabRn9BJRJc-3ksC39hwSs3ZzG9B8  PeACz8rabRn9BJRJc  3ksC39hwSs3ZzG9B8   \n",
       "1   Pj9mwxXYPKxjPaxST-mgoxZfxyary28EXFn  Pj9mwxXYPKxjPaxST  mgoxZfxyary28EXFn   \n",
       "2   9mdkMmj4pY8Q2TwqF-P8SGjH7mLk9FxCf45  9mdkMmj4pY8Q2TwqF  P8SGjH7mLk9FxCf45   \n",
       "3   9mdkMmj4pY8Q2TwqF-WNJjNxKHc5ubX2jwc  9mdkMmj4pY8Q2TwqF  WNJjNxKHc5ubX2jwc   \n",
       "4   BibLRuKtNNv7QEDqb-jdfB7xy4iuaG66yiZ  BibLRuKtNNv7QEDqb  jdfB7xy4iuaG66yiZ   \n",
       "5   9mdkMmj4pY8Q2TwqF-hL8oByh2FnRJSk7LL  9mdkMmj4pY8Q2TwqF  hL8oByh2FnRJSk7LL   \n",
       "6   9mdkMmj4pY8Q2TwqF-mKv8rBNJhkFbmSarX  9mdkMmj4pY8Q2TwqF  mKv8rBNJhkFbmSarX   \n",
       "7   sM3z5FkZfsABqcj3g-4dEvFqkiADYDtSNeR  sM3z5FkZfsABqcj3g  4dEvFqkiADYDtSNeR   \n",
       "8   9mdkMmj4pY8Q2TwqF-qcYXZGnrw3cmEsL5v  9mdkMmj4pY8Q2TwqF  qcYXZGnrw3cmEsL5v   \n",
       "9   BibLRuKtNNv7QEDqb-r7cB6GdSpMjLpK8Jc  BibLRuKtNNv7QEDqb  r7cB6GdSpMjLpK8Jc   \n",
       "10  BibLRuKtNNv7QEDqb-Q9qoS4WQTWAEw47NH  BibLRuKtNNv7QEDqb  Q9qoS4WQTWAEw47NH   \n",
       "11  BibLRuKtNNv7QEDqb-QewbaRfokgfMFiD5C  BibLRuKtNNv7QEDqb  QewbaRfokgfMFiD5C   \n",
       "12  cToFEbgXcFbrKsSrj-o33KWpqZFwSNt3tDe  cToFEbgXcFbrKsSrj  o33KWpqZFwSNt3tDe   \n",
       "13  BibLRuKtNNv7QEDqb-YuHJWnrgg2prjPjRp  BibLRuKtNNv7QEDqb  YuHJWnrgg2prjPjRp   \n",
       "14  BibLRuKtNNv7QEDqb-evGbShCDMHxvCBABe  BibLRuKtNNv7QEDqb  evGbShCDMHxvCBABe   \n",
       "15  ckNqEpoM9jbCER9ex-qNMTGWnYiCzfSdRzv  ckNqEpoM9jbCER9ex  qNMTGWnYiCzfSdRzv   \n",
       "16  sM3z5FkZfsABqcj3g-CJEjiJ2v3Wmq7SXnK  sM3z5FkZfsABqcj3g  CJEjiJ2v3Wmq7SXnK   \n",
       "17  BibLRuKtNNv7QEDqb-mMf4jK8LZ9cBY7yJK  BibLRuKtNNv7QEDqb  mMf4jK8LZ9cBY7yJK   \n",
       "18  PeACz8rabRn9BJRJc-w8mMv9X4eQe8y7tmG  PeACz8rabRn9BJRJc  w8mMv9X4eQe8y7tmG   \n",
       "19  sM3z5FkZfsABqcj3g-jZaAYDh4ZgYESGsT3  sM3z5FkZfsABqcj3g  jZaAYDh4ZgYESGsT3   \n",
       "20  sM3z5FkZfsABqcj3g-wrkkbAdZf9AMeKgsH  sM3z5FkZfsABqcj3g  wrkkbAdZf9AMeKgsH   \n",
       "21  Pj9mwxXYPKxjPaxST-MmfKpPtSv5TWDy8Jc  Pj9mwxXYPKxjPaxST  MmfKpPtSv5TWDy8Jc   \n",
       "22  cToFEbgXcFbrKsSrj-9LidQoQiuSXdwJwoc  cToFEbgXcFbrKsSrj  9LidQoQiuSXdwJwoc   \n",
       "23  cToFEbgXcFbrKsSrj-Kevyvh5GoYBa6QhtS  cToFEbgXcFbrKsSrj  Kevyvh5GoYBa6QhtS   \n",
       "24  cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt  cToFEbgXcFbrKsSrj  kTtxafCNsKoqsnhCt   \n",
       "25  fduEdDA8nk5ybcYze-9deZN5msXPWbRbaty  fduEdDA8nk5ybcYze  9deZN5msXPWbRbaty   \n",
       "26  fduEdDA8nk5ybcYze-GjXnW5cj2SE8rzNCJ  fduEdDA8nk5ybcYze  GjXnW5cj2SE8rzNCJ   \n",
       "27  fduEdDA8nk5ybcYze-vyrrtwEpzzRqCACxi  fduEdDA8nk5ybcYze  vyrrtwEpzzRqCACxi   \n",
       "28  9mdkMmj4pY8Q2TwqF-nRAQpsPhsQs4zRvTL  9mdkMmj4pY8Q2TwqF  nRAQpsPhsQs4zRvTL   \n",
       "29  tAC6QTAqiEgFRpHMj-7JhveY48LSxc8s87j  tAC6QTAqiEgFRpHMj  7JhveY48LSxc8s87j   \n",
       "30  FmHT4pnqrPXzCLCmE-z8eGJ6ktY8jpBaAXN  FmHT4pnqrPXzCLCmE  z8eGJ6ktY8jpBaAXN   \n",
       "31  9mdkMmj4pY8Q2TwqF-Q2Qffh22oZwWrRCWt  9mdkMmj4pY8Q2TwqF  Q2Qffh22oZwWrRCWt   \n",
       "32  FmHT4pnqrPXzCLCmE-EbjzaxZ6wADuYQKqq  FmHT4pnqrPXzCLCmE  EbjzaxZ6wADuYQKqq   \n",
       "33  qyiCF6mPcE2x9rQFa-452ZTHfYTnMazmiEg  qyiCF6mPcE2x9rQFa  452ZTHfYTnMazmiEg   \n",
       "34  Pj9mwxXYPKxjPaxST-Tho38bGKWrSDgZbJA  Pj9mwxXYPKxjPaxST  Tho38bGKWrSDgZbJA   \n",
       "35  KaYHdgPnyHQx84CPL-cqPJEg45npP4KZZqS  KaYHdgPnyHQx84CPL  cqPJEg45npP4KZZqS   \n",
       "36  Pj9mwxXYPKxjPaxST-ZEgMQhsGM5LM8HTJ7  Pj9mwxXYPKxjPaxST  ZEgMQhsGM5LM8HTJ7   \n",
       "37  Pj9mwxXYPKxjPaxST-kDNiwKsweERmTmqqR  Pj9mwxXYPKxjPaxST  kDNiwKsweERmTmqqR   \n",
       "\n",
       "   conditions_failed  user_count  task_count  expected_task_count  \\\n",
       "0   [1, 2]            14.0        14.0        14.0                  \n",
       "1   [1, 2]            8.0         8.0         8.0                   \n",
       "2   [1, 2]            12.0        12.0        12.0                  \n",
       "3   [1, 2]            16.0        16.0        16.0                  \n",
       "4   [1, 2]            8.0         8.0         8.0                   \n",
       "5   [1, 2]            10.0        10.0        10.0                  \n",
       "6   [1, 2]            16.0        16.0        16.0                  \n",
       "7   [1, 2]            10.0        10.0        10.0                  \n",
       "8   [1, 2]            16.0        16.0        16.0                  \n",
       "9   [1, 2]            10.0        10.0        10.0                  \n",
       "10  [1, 2]            4.0         4.0         4.0                   \n",
       "11  [1, 2]            18.0        18.0        18.0                  \n",
       "12  [1, 2]            10.0        10.0        10.0                  \n",
       "13  [1, 2]            4.0         4.0         4.0                   \n",
       "14  [1, 2]            6.0         6.0         6.0                   \n",
       "15  [1, 2]            14.0        14.0        14.0                  \n",
       "16  [1, 2]            10.0        10.0        10.0                  \n",
       "17  [1, 2]            8.0         8.0         8.0                   \n",
       "18  [1, 2]            4.0         4.0         4.0                   \n",
       "19  [1, 2]            14.0        14.0        14.0                  \n",
       "20  [1, 2]            10.0        10.0        10.0                  \n",
       "21  [1, 2, 3]         8.0         0.0         8.0                   \n",
       "22  [1, 2, 3]         14.0        7.0         14.0                  \n",
       "23  [1, 2, 3]         14.0        7.0         14.0                  \n",
       "24  [1, 2, 3]         14.0        7.0         14.0                  \n",
       "25  [1, 2, 3]         3.0         0.0         3.0                   \n",
       "26  [1, 2, 3]         3.0         0.0         3.0                   \n",
       "27  [1, 2, 3]         3.0         0.0         3.0                   \n",
       "28  [1, 2, 3]         10.0        24.0        10.0                  \n",
       "29  [1, 2, 3]         2.0         0.0         2.0                   \n",
       "30  [1, 2, 3]         2.0         0.0         2.0                   \n",
       "31  [1, 2, 3]         8.0         32.0        8.0                   \n",
       "32  [1, 2, 3]         2.0         0.0         2.0                   \n",
       "33  [1, 2, 3]         2.0         0.0         2.0                   \n",
       "34  [1, 2, 3]         8.0         0.0         8.0                   \n",
       "35  [1, 2, 3]         8.0         6.0         8.0                   \n",
       "36  [1, 2, 3]         8.0         0.0         8.0                   \n",
       "37  [1, 2, 3]         8.0         0.0         8.0                   \n",
       "\n",
       "    pairing_count  expected_pairing_count  pairs_hist_count  \\\n",
       "0   8.0            7.0                     8.0                \n",
       "1   5.0            4.0                     5.0                \n",
       "2   7.0            6.0                     7.0                \n",
       "3   9.0            8.0                     9.0                \n",
       "4   5.0            4.0                     5.0                \n",
       "5   6.0            5.0                     6.0                \n",
       "6   9.0            8.0                     9.0                \n",
       "7   6.0            5.0                     6.0                \n",
       "8   9.0            8.0                     9.0                \n",
       "9   6.0            5.0                     6.0                \n",
       "10  3.0            2.0                     3.0                \n",
       "11  10.0           9.0                     10.0               \n",
       "12  6.0            5.0                     6.0                \n",
       "13  3.0            2.0                     3.0                \n",
       "14  4.0            3.0                     4.0                \n",
       "15  8.0            7.0                     8.0                \n",
       "16  6.0            5.0                     6.0                \n",
       "17  5.0            4.0                     5.0                \n",
       "18  3.0            2.0                     3.0                \n",
       "19  8.0            7.0                     8.0                \n",
       "20  6.0            5.0                     6.0                \n",
       "21  5.0            4.0                     5.0                \n",
       "22  8.0            7.0                     8.0                \n",
       "23  8.0            7.0                     8.0                \n",
       "24  8.0            7.0                     8.0                \n",
       "25  3.0            2.0                     3.0                \n",
       "26  3.0            2.0                     3.0                \n",
       "27  3.0            2.0                     3.0                \n",
       "28  6.0            5.0                     24.0               \n",
       "29  2.0            1.0                     2.0                \n",
       "30  2.0            1.0                     2.0                \n",
       "31  5.0            4.0                     20.0               \n",
       "32  2.0            1.0                     2.0                \n",
       "33  2.0            1.0                     2.0                \n",
       "34  5.0            4.0                     5.0                \n",
       "35  5.0            4.0                     5.0                \n",
       "36  5.0            4.0                     5.0                \n",
       "37  6.0            4.0                     6.0                \n",
       "\n",
       "    expected_pairs_hist_count  affinity_count  expected_affinity_count  \n",
       "0   7.0                        109.0           182.0                    \n",
       "1   4.0                        48.0            56.0                     \n",
       "2   6.0                        132.0           132.0                    \n",
       "3   8.0                        234.0           240.0                    \n",
       "4   4.0                        49.0            56.0                     \n",
       "5   5.0                        90.0            90.0                     \n",
       "6   8.0                        234.0           240.0                    \n",
       "7   5.0                        82.0            90.0                     \n",
       "8   8.0                        240.0           240.0                    \n",
       "9   5.0                        90.0            90.0                     \n",
       "10  2.0                        12.0            12.0                     \n",
       "11  9.0                        306.0           306.0                    \n",
       "12  5.0                        69.0            90.0                     \n",
       "13  2.0                        8.0             12.0                     \n",
       "14  3.0                        29.0            30.0                     \n",
       "15  7.0                        182.0           182.0                    \n",
       "16  5.0                        82.0            90.0                     \n",
       "17  4.0                        55.0            56.0                     \n",
       "18  2.0                        6.0             12.0                     \n",
       "19  7.0                        155.0           182.0                    \n",
       "20  5.0                        85.0            90.0                     \n",
       "21  4.0                        55.0            56.0                     \n",
       "22  7.0                        134.0           182.0                    \n",
       "23  7.0                        133.0           182.0                    \n",
       "24  7.0                        131.0           182.0                    \n",
       "25  2.0                        6.0             6.0                      \n",
       "26  2.0                        6.0             6.0                      \n",
       "27  2.0                        6.0             6.0                      \n",
       "28  5.0                        30.0            90.0                     \n",
       "29  1.0                        2.0             2.0                      \n",
       "30  1.0                        2.0             2.0                      \n",
       "31  4.0                        56.0            56.0                     \n",
       "32  1.0                        1.0             2.0                      \n",
       "33  1.0                        2.0             2.0                      \n",
       "34  4.0                        55.0            56.0                     \n",
       "35  4.0                        56.0            56.0                     \n",
       "36  4.0                        55.0            56.0                     \n",
       "37  4.0                        55.0            56.0                     "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "invalid_group_pairings[invalid_group_pairings.conditions_failed.apply(lambda x: '1' in x)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>group_pairing_id</th>\n",
       "      <th>group_id</th>\n",
       "      <th>pairing_id</th>\n",
       "      <th>conditions_failed</th>\n",
       "      <th>user_count</th>\n",
       "      <th>task_count</th>\n",
       "      <th>expected_task_count</th>\n",
       "      <th>pairing_count</th>\n",
       "      <th>expected_pairing_count</th>\n",
       "      <th>pairs_hist_count</th>\n",
       "      <th>expected_pairs_hist_count</th>\n",
       "      <th>affinity_count</th>\n",
       "      <th>expected_affinity_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>PeACz8rabRn9BJRJc-3ksC39hwSs3ZzG9B8</td>\n",
       "      <td>PeACz8rabRn9BJRJc</td>\n",
       "      <td>3ksC39hwSs3ZzG9B8</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>14.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>109.0</td>\n",
       "      <td>182.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Pj9mwxXYPKxjPaxST-mgoxZfxyary28EXFn</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST</td>\n",
       "      <td>mgoxZfxyary28EXFn</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>56.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-P8SGjH7mLk9FxCf45</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF</td>\n",
       "      <td>P8SGjH7mLk9FxCf45</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>12.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>132.0</td>\n",
       "      <td>132.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-WNJjNxKHc5ubX2jwc</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF</td>\n",
       "      <td>WNJjNxKHc5ubX2jwc</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>16.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>234.0</td>\n",
       "      <td>240.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>BibLRuKtNNv7QEDqb-jdfB7xy4iuaG66yiZ</td>\n",
       "      <td>BibLRuKtNNv7QEDqb</td>\n",
       "      <td>jdfB7xy4iuaG66yiZ</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>56.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-hL8oByh2FnRJSk7LL</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF</td>\n",
       "      <td>hL8oByh2FnRJSk7LL</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>90.0</td>\n",
       "      <td>90.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-mKv8rBNJhkFbmSarX</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF</td>\n",
       "      <td>mKv8rBNJhkFbmSarX</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>16.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>234.0</td>\n",
       "      <td>240.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-4dEvFqkiADYDtSNeR</td>\n",
       "      <td>sM3z5FkZfsABqcj3g</td>\n",
       "      <td>4dEvFqkiADYDtSNeR</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>82.0</td>\n",
       "      <td>90.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-qcYXZGnrw3cmEsL5v</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF</td>\n",
       "      <td>qcYXZGnrw3cmEsL5v</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>16.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>240.0</td>\n",
       "      <td>240.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>BibLRuKtNNv7QEDqb-r7cB6GdSpMjLpK8Jc</td>\n",
       "      <td>BibLRuKtNNv7QEDqb</td>\n",
       "      <td>r7cB6GdSpMjLpK8Jc</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>90.0</td>\n",
       "      <td>90.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>BibLRuKtNNv7QEDqb-Q9qoS4WQTWAEw47NH</td>\n",
       "      <td>BibLRuKtNNv7QEDqb</td>\n",
       "      <td>Q9qoS4WQTWAEw47NH</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>12.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>BibLRuKtNNv7QEDqb-QewbaRfokgfMFiD5C</td>\n",
       "      <td>BibLRuKtNNv7QEDqb</td>\n",
       "      <td>QewbaRfokgfMFiD5C</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>18.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>306.0</td>\n",
       "      <td>306.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>cToFEbgXcFbrKsSrj-o33KWpqZFwSNt3tDe</td>\n",
       "      <td>cToFEbgXcFbrKsSrj</td>\n",
       "      <td>o33KWpqZFwSNt3tDe</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>69.0</td>\n",
       "      <td>90.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>BibLRuKtNNv7QEDqb-YuHJWnrgg2prjPjRp</td>\n",
       "      <td>BibLRuKtNNv7QEDqb</td>\n",
       "      <td>YuHJWnrgg2prjPjRp</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>12.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>BibLRuKtNNv7QEDqb-evGbShCDMHxvCBABe</td>\n",
       "      <td>BibLRuKtNNv7QEDqb</td>\n",
       "      <td>evGbShCDMHxvCBABe</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>30.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>ckNqEpoM9jbCER9ex-qNMTGWnYiCzfSdRzv</td>\n",
       "      <td>ckNqEpoM9jbCER9ex</td>\n",
       "      <td>qNMTGWnYiCzfSdRzv</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>14.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>182.0</td>\n",
       "      <td>182.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-CJEjiJ2v3Wmq7SXnK</td>\n",
       "      <td>sM3z5FkZfsABqcj3g</td>\n",
       "      <td>CJEjiJ2v3Wmq7SXnK</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>82.0</td>\n",
       "      <td>90.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>BibLRuKtNNv7QEDqb-mMf4jK8LZ9cBY7yJK</td>\n",
       "      <td>BibLRuKtNNv7QEDqb</td>\n",
       "      <td>mMf4jK8LZ9cBY7yJK</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>55.0</td>\n",
       "      <td>56.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>PeACz8rabRn9BJRJc-w8mMv9X4eQe8y7tmG</td>\n",
       "      <td>PeACz8rabRn9BJRJc</td>\n",
       "      <td>w8mMv9X4eQe8y7tmG</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>12.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-jZaAYDh4ZgYESGsT3</td>\n",
       "      <td>sM3z5FkZfsABqcj3g</td>\n",
       "      <td>jZaAYDh4ZgYESGsT3</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>14.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>155.0</td>\n",
       "      <td>182.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-wrkkbAdZf9AMeKgsH</td>\n",
       "      <td>sM3z5FkZfsABqcj3g</td>\n",
       "      <td>wrkkbAdZf9AMeKgsH</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>85.0</td>\n",
       "      <td>90.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Pj9mwxXYPKxjPaxST-MmfKpPtSv5TWDy8Jc</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST</td>\n",
       "      <td>MmfKpPtSv5TWDy8Jc</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>55.0</td>\n",
       "      <td>56.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>cToFEbgXcFbrKsSrj-9LidQoQiuSXdwJwoc</td>\n",
       "      <td>cToFEbgXcFbrKsSrj</td>\n",
       "      <td>9LidQoQiuSXdwJwoc</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>14.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>134.0</td>\n",
       "      <td>182.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>cToFEbgXcFbrKsSrj-Kevyvh5GoYBa6QhtS</td>\n",
       "      <td>cToFEbgXcFbrKsSrj</td>\n",
       "      <td>Kevyvh5GoYBa6QhtS</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>14.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>133.0</td>\n",
       "      <td>182.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt</td>\n",
       "      <td>cToFEbgXcFbrKsSrj</td>\n",
       "      <td>kTtxafCNsKoqsnhCt</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>14.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>131.0</td>\n",
       "      <td>182.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>fduEdDA8nk5ybcYze-9deZN5msXPWbRbaty</td>\n",
       "      <td>fduEdDA8nk5ybcYze</td>\n",
       "      <td>9deZN5msXPWbRbaty</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>fduEdDA8nk5ybcYze-GjXnW5cj2SE8rzNCJ</td>\n",
       "      <td>fduEdDA8nk5ybcYze</td>\n",
       "      <td>GjXnW5cj2SE8rzNCJ</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>fduEdDA8nk5ybcYze-vyrrtwEpzzRqCACxi</td>\n",
       "      <td>fduEdDA8nk5ybcYze</td>\n",
       "      <td>vyrrtwEpzzRqCACxi</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-nRAQpsPhsQs4zRvTL</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF</td>\n",
       "      <td>nRAQpsPhsQs4zRvTL</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>10.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>90.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>tAC6QTAqiEgFRpHMj-7JhveY48LSxc8s87j</td>\n",
       "      <td>tAC6QTAqiEgFRpHMj</td>\n",
       "      <td>7JhveY48LSxc8s87j</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>FmHT4pnqrPXzCLCmE-z8eGJ6ktY8jpBaAXN</td>\n",
       "      <td>FmHT4pnqrPXzCLCmE</td>\n",
       "      <td>z8eGJ6ktY8jpBaAXN</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-Q2Qffh22oZwWrRCWt</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF</td>\n",
       "      <td>Q2Qffh22oZwWrRCWt</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>8.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>56.0</td>\n",
       "      <td>56.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>FmHT4pnqrPXzCLCmE-EbjzaxZ6wADuYQKqq</td>\n",
       "      <td>FmHT4pnqrPXzCLCmE</td>\n",
       "      <td>EbjzaxZ6wADuYQKqq</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>qyiCF6mPcE2x9rQFa-452ZTHfYTnMazmiEg</td>\n",
       "      <td>qyiCF6mPcE2x9rQFa</td>\n",
       "      <td>452ZTHfYTnMazmiEg</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>Pj9mwxXYPKxjPaxST-Tho38bGKWrSDgZbJA</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST</td>\n",
       "      <td>Tho38bGKWrSDgZbJA</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>55.0</td>\n",
       "      <td>56.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>KaYHdgPnyHQx84CPL-cqPJEg45npP4KZZqS</td>\n",
       "      <td>KaYHdgPnyHQx84CPL</td>\n",
       "      <td>cqPJEg45npP4KZZqS</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>8.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>56.0</td>\n",
       "      <td>56.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>Pj9mwxXYPKxjPaxST-ZEgMQhsGM5LM8HTJ7</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST</td>\n",
       "      <td>ZEgMQhsGM5LM8HTJ7</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>55.0</td>\n",
       "      <td>56.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>Pj9mwxXYPKxjPaxST-kDNiwKsweERmTmqqR</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST</td>\n",
       "      <td>kDNiwKsweERmTmqqR</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>55.0</td>\n",
       "      <td>56.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-ctPEz48CJqcA54YeD</td>\n",
       "      <td>9mdkMmj4pY8Q2TwqF</td>\n",
       "      <td>ctPEz48CJqcA54YeD</td>\n",
       "      <td>[2, 3]</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       group_pairing_id           group_id         pairing_id  \\\n",
       "0   PeACz8rabRn9BJRJc-3ksC39hwSs3ZzG9B8  PeACz8rabRn9BJRJc  3ksC39hwSs3ZzG9B8   \n",
       "1   Pj9mwxXYPKxjPaxST-mgoxZfxyary28EXFn  Pj9mwxXYPKxjPaxST  mgoxZfxyary28EXFn   \n",
       "2   9mdkMmj4pY8Q2TwqF-P8SGjH7mLk9FxCf45  9mdkMmj4pY8Q2TwqF  P8SGjH7mLk9FxCf45   \n",
       "3   9mdkMmj4pY8Q2TwqF-WNJjNxKHc5ubX2jwc  9mdkMmj4pY8Q2TwqF  WNJjNxKHc5ubX2jwc   \n",
       "4   BibLRuKtNNv7QEDqb-jdfB7xy4iuaG66yiZ  BibLRuKtNNv7QEDqb  jdfB7xy4iuaG66yiZ   \n",
       "5   9mdkMmj4pY8Q2TwqF-hL8oByh2FnRJSk7LL  9mdkMmj4pY8Q2TwqF  hL8oByh2FnRJSk7LL   \n",
       "6   9mdkMmj4pY8Q2TwqF-mKv8rBNJhkFbmSarX  9mdkMmj4pY8Q2TwqF  mKv8rBNJhkFbmSarX   \n",
       "7   sM3z5FkZfsABqcj3g-4dEvFqkiADYDtSNeR  sM3z5FkZfsABqcj3g  4dEvFqkiADYDtSNeR   \n",
       "8   9mdkMmj4pY8Q2TwqF-qcYXZGnrw3cmEsL5v  9mdkMmj4pY8Q2TwqF  qcYXZGnrw3cmEsL5v   \n",
       "9   BibLRuKtNNv7QEDqb-r7cB6GdSpMjLpK8Jc  BibLRuKtNNv7QEDqb  r7cB6GdSpMjLpK8Jc   \n",
       "10  BibLRuKtNNv7QEDqb-Q9qoS4WQTWAEw47NH  BibLRuKtNNv7QEDqb  Q9qoS4WQTWAEw47NH   \n",
       "11  BibLRuKtNNv7QEDqb-QewbaRfokgfMFiD5C  BibLRuKtNNv7QEDqb  QewbaRfokgfMFiD5C   \n",
       "12  cToFEbgXcFbrKsSrj-o33KWpqZFwSNt3tDe  cToFEbgXcFbrKsSrj  o33KWpqZFwSNt3tDe   \n",
       "13  BibLRuKtNNv7QEDqb-YuHJWnrgg2prjPjRp  BibLRuKtNNv7QEDqb  YuHJWnrgg2prjPjRp   \n",
       "14  BibLRuKtNNv7QEDqb-evGbShCDMHxvCBABe  BibLRuKtNNv7QEDqb  evGbShCDMHxvCBABe   \n",
       "15  ckNqEpoM9jbCER9ex-qNMTGWnYiCzfSdRzv  ckNqEpoM9jbCER9ex  qNMTGWnYiCzfSdRzv   \n",
       "16  sM3z5FkZfsABqcj3g-CJEjiJ2v3Wmq7SXnK  sM3z5FkZfsABqcj3g  CJEjiJ2v3Wmq7SXnK   \n",
       "17  BibLRuKtNNv7QEDqb-mMf4jK8LZ9cBY7yJK  BibLRuKtNNv7QEDqb  mMf4jK8LZ9cBY7yJK   \n",
       "18  PeACz8rabRn9BJRJc-w8mMv9X4eQe8y7tmG  PeACz8rabRn9BJRJc  w8mMv9X4eQe8y7tmG   \n",
       "19  sM3z5FkZfsABqcj3g-jZaAYDh4ZgYESGsT3  sM3z5FkZfsABqcj3g  jZaAYDh4ZgYESGsT3   \n",
       "20  sM3z5FkZfsABqcj3g-wrkkbAdZf9AMeKgsH  sM3z5FkZfsABqcj3g  wrkkbAdZf9AMeKgsH   \n",
       "21  Pj9mwxXYPKxjPaxST-MmfKpPtSv5TWDy8Jc  Pj9mwxXYPKxjPaxST  MmfKpPtSv5TWDy8Jc   \n",
       "22  cToFEbgXcFbrKsSrj-9LidQoQiuSXdwJwoc  cToFEbgXcFbrKsSrj  9LidQoQiuSXdwJwoc   \n",
       "23  cToFEbgXcFbrKsSrj-Kevyvh5GoYBa6QhtS  cToFEbgXcFbrKsSrj  Kevyvh5GoYBa6QhtS   \n",
       "24  cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt  cToFEbgXcFbrKsSrj  kTtxafCNsKoqsnhCt   \n",
       "25  fduEdDA8nk5ybcYze-9deZN5msXPWbRbaty  fduEdDA8nk5ybcYze  9deZN5msXPWbRbaty   \n",
       "26  fduEdDA8nk5ybcYze-GjXnW5cj2SE8rzNCJ  fduEdDA8nk5ybcYze  GjXnW5cj2SE8rzNCJ   \n",
       "27  fduEdDA8nk5ybcYze-vyrrtwEpzzRqCACxi  fduEdDA8nk5ybcYze  vyrrtwEpzzRqCACxi   \n",
       "28  9mdkMmj4pY8Q2TwqF-nRAQpsPhsQs4zRvTL  9mdkMmj4pY8Q2TwqF  nRAQpsPhsQs4zRvTL   \n",
       "29  tAC6QTAqiEgFRpHMj-7JhveY48LSxc8s87j  tAC6QTAqiEgFRpHMj  7JhveY48LSxc8s87j   \n",
       "30  FmHT4pnqrPXzCLCmE-z8eGJ6ktY8jpBaAXN  FmHT4pnqrPXzCLCmE  z8eGJ6ktY8jpBaAXN   \n",
       "31  9mdkMmj4pY8Q2TwqF-Q2Qffh22oZwWrRCWt  9mdkMmj4pY8Q2TwqF  Q2Qffh22oZwWrRCWt   \n",
       "32  FmHT4pnqrPXzCLCmE-EbjzaxZ6wADuYQKqq  FmHT4pnqrPXzCLCmE  EbjzaxZ6wADuYQKqq   \n",
       "33  qyiCF6mPcE2x9rQFa-452ZTHfYTnMazmiEg  qyiCF6mPcE2x9rQFa  452ZTHfYTnMazmiEg   \n",
       "34  Pj9mwxXYPKxjPaxST-Tho38bGKWrSDgZbJA  Pj9mwxXYPKxjPaxST  Tho38bGKWrSDgZbJA   \n",
       "35  KaYHdgPnyHQx84CPL-cqPJEg45npP4KZZqS  KaYHdgPnyHQx84CPL  cqPJEg45npP4KZZqS   \n",
       "36  Pj9mwxXYPKxjPaxST-ZEgMQhsGM5LM8HTJ7  Pj9mwxXYPKxjPaxST  ZEgMQhsGM5LM8HTJ7   \n",
       "37  Pj9mwxXYPKxjPaxST-kDNiwKsweERmTmqqR  Pj9mwxXYPKxjPaxST  kDNiwKsweERmTmqqR   \n",
       "38  9mdkMmj4pY8Q2TwqF-ctPEz48CJqcA54YeD  9mdkMmj4pY8Q2TwqF  ctPEz48CJqcA54YeD   \n",
       "\n",
       "   conditions_failed  user_count  task_count  expected_task_count  \\\n",
       "0   [1, 2]            14.0        14.0        14.0                  \n",
       "1   [1, 2]            8.0         8.0         8.0                   \n",
       "2   [1, 2]            12.0        12.0        12.0                  \n",
       "3   [1, 2]            16.0        16.0        16.0                  \n",
       "4   [1, 2]            8.0         8.0         8.0                   \n",
       "5   [1, 2]            10.0        10.0        10.0                  \n",
       "6   [1, 2]            16.0        16.0        16.0                  \n",
       "7   [1, 2]            10.0        10.0        10.0                  \n",
       "8   [1, 2]            16.0        16.0        16.0                  \n",
       "9   [1, 2]            10.0        10.0        10.0                  \n",
       "10  [1, 2]            4.0         4.0         4.0                   \n",
       "11  [1, 2]            18.0        18.0        18.0                  \n",
       "12  [1, 2]            10.0        10.0        10.0                  \n",
       "13  [1, 2]            4.0         4.0         4.0                   \n",
       "14  [1, 2]            6.0         6.0         6.0                   \n",
       "15  [1, 2]            14.0        14.0        14.0                  \n",
       "16  [1, 2]            10.0        10.0        10.0                  \n",
       "17  [1, 2]            8.0         8.0         8.0                   \n",
       "18  [1, 2]            4.0         4.0         4.0                   \n",
       "19  [1, 2]            14.0        14.0        14.0                  \n",
       "20  [1, 2]            10.0        10.0        10.0                  \n",
       "21  [1, 2, 3]         8.0         0.0         8.0                   \n",
       "22  [1, 2, 3]         14.0        7.0         14.0                  \n",
       "23  [1, 2, 3]         14.0        7.0         14.0                  \n",
       "24  [1, 2, 3]         14.0        7.0         14.0                  \n",
       "25  [1, 2, 3]         3.0         0.0         3.0                   \n",
       "26  [1, 2, 3]         3.0         0.0         3.0                   \n",
       "27  [1, 2, 3]         3.0         0.0         3.0                   \n",
       "28  [1, 2, 3]         10.0        24.0        10.0                  \n",
       "29  [1, 2, 3]         2.0         0.0         2.0                   \n",
       "30  [1, 2, 3]         2.0         0.0         2.0                   \n",
       "31  [1, 2, 3]         8.0         32.0        8.0                   \n",
       "32  [1, 2, 3]         2.0         0.0         2.0                   \n",
       "33  [1, 2, 3]         2.0         0.0         2.0                   \n",
       "34  [1, 2, 3]         8.0         0.0         8.0                   \n",
       "35  [1, 2, 3]         8.0         6.0         8.0                   \n",
       "36  [1, 2, 3]         8.0         0.0         8.0                   \n",
       "37  [1, 2, 3]         8.0         0.0         8.0                   \n",
       "38  [2, 3]            2.0         0.0         2.0                   \n",
       "\n",
       "    pairing_count  expected_pairing_count  pairs_hist_count  \\\n",
       "0   8.0            7.0                     8.0                \n",
       "1   5.0            4.0                     5.0                \n",
       "2   7.0            6.0                     7.0                \n",
       "3   9.0            8.0                     9.0                \n",
       "4   5.0            4.0                     5.0                \n",
       "5   6.0            5.0                     6.0                \n",
       "6   9.0            8.0                     9.0                \n",
       "7   6.0            5.0                     6.0                \n",
       "8   9.0            8.0                     9.0                \n",
       "9   6.0            5.0                     6.0                \n",
       "10  3.0            2.0                     3.0                \n",
       "11  10.0           9.0                     10.0               \n",
       "12  6.0            5.0                     6.0                \n",
       "13  3.0            2.0                     3.0                \n",
       "14  4.0            3.0                     4.0                \n",
       "15  8.0            7.0                     8.0                \n",
       "16  6.0            5.0                     6.0                \n",
       "17  5.0            4.0                     5.0                \n",
       "18  3.0            2.0                     3.0                \n",
       "19  8.0            7.0                     8.0                \n",
       "20  6.0            5.0                     6.0                \n",
       "21  5.0            4.0                     5.0                \n",
       "22  8.0            7.0                     8.0                \n",
       "23  8.0            7.0                     8.0                \n",
       "24  8.0            7.0                     8.0                \n",
       "25  3.0            2.0                     3.0                \n",
       "26  3.0            2.0                     3.0                \n",
       "27  3.0            2.0                     3.0                \n",
       "28  6.0            5.0                     24.0               \n",
       "29  2.0            1.0                     2.0                \n",
       "30  2.0            1.0                     2.0                \n",
       "31  5.0            4.0                     20.0               \n",
       "32  2.0            1.0                     2.0                \n",
       "33  2.0            1.0                     2.0                \n",
       "34  5.0            4.0                     5.0                \n",
       "35  5.0            4.0                     5.0                \n",
       "36  5.0            4.0                     5.0                \n",
       "37  6.0            4.0                     6.0                \n",
       "38  1.0            1.0                     4.0                \n",
       "\n",
       "    expected_pairs_hist_count  affinity_count  expected_affinity_count  \n",
       "0   7.0                        109.0           182.0                    \n",
       "1   4.0                        48.0            56.0                     \n",
       "2   6.0                        132.0           132.0                    \n",
       "3   8.0                        234.0           240.0                    \n",
       "4   4.0                        49.0            56.0                     \n",
       "5   5.0                        90.0            90.0                     \n",
       "6   8.0                        234.0           240.0                    \n",
       "7   5.0                        82.0            90.0                     \n",
       "8   8.0                        240.0           240.0                    \n",
       "9   5.0                        90.0            90.0                     \n",
       "10  2.0                        12.0            12.0                     \n",
       "11  9.0                        306.0           306.0                    \n",
       "12  5.0                        69.0            90.0                     \n",
       "13  2.0                        8.0             12.0                     \n",
       "14  3.0                        29.0            30.0                     \n",
       "15  7.0                        182.0           182.0                    \n",
       "16  5.0                        82.0            90.0                     \n",
       "17  4.0                        55.0            56.0                     \n",
       "18  2.0                        6.0             12.0                     \n",
       "19  7.0                        155.0           182.0                    \n",
       "20  5.0                        85.0            90.0                     \n",
       "21  4.0                        55.0            56.0                     \n",
       "22  7.0                        134.0           182.0                    \n",
       "23  7.0                        133.0           182.0                    \n",
       "24  7.0                        131.0           182.0                    \n",
       "25  2.0                        6.0             6.0                      \n",
       "26  2.0                        6.0             6.0                      \n",
       "27  2.0                        6.0             6.0                      \n",
       "28  5.0                        30.0            90.0                     \n",
       "29  1.0                        2.0             2.0                      \n",
       "30  1.0                        2.0             2.0                      \n",
       "31  4.0                        56.0            56.0                     \n",
       "32  1.0                        1.0             2.0                      \n",
       "33  1.0                        2.0             2.0                      \n",
       "34  4.0                        55.0            56.0                     \n",
       "35  4.0                        56.0            56.0                     \n",
       "36  4.0                        55.0            56.0                     \n",
       "37  4.0                        55.0            56.0                     \n",
       "38  1.0                        0.0             2.0                      "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "invalid_group_pairings[invalid_group_pairings.conditions_failed.apply(lambda x: '2' in x)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TODO: handle case where two unpaired people are in the pairing (this is a bug in the meteor code)\n",
    "\n",
    "# TODO: are tasks appearing in tasks but not in tasks_history? maybe some groups are missing tasks because they aren't archiving properly?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "group_pair_id_tester = \"Pj9mwxXYPKxjPaxST-kDNiwKsweERmTmqqR\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>group_pairing_id</th>\n",
       "      <th>group_id</th>\n",
       "      <th>pairing_id</th>\n",
       "      <th>conditions_failed</th>\n",
       "      <th>user_count</th>\n",
       "      <th>task_count</th>\n",
       "      <th>expected_task_count</th>\n",
       "      <th>pairing_count</th>\n",
       "      <th>expected_pairing_count</th>\n",
       "      <th>pairs_hist_count</th>\n",
       "      <th>expected_pairs_hist_count</th>\n",
       "      <th>affinity_count</th>\n",
       "      <th>expected_affinity_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>Pj9mwxXYPKxjPaxST-kDNiwKsweERmTmqqR</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST</td>\n",
       "      <td>kDNiwKsweERmTmqqR</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>55.0</td>\n",
       "      <td>56.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       group_pairing_id           group_id         pairing_id  \\\n",
       "37  Pj9mwxXYPKxjPaxST-kDNiwKsweERmTmqqR  Pj9mwxXYPKxjPaxST  kDNiwKsweERmTmqqR   \n",
       "\n",
       "   conditions_failed  user_count  task_count  expected_task_count  \\\n",
       "37  [1, 2, 3]         8.0         0.0         8.0                   \n",
       "\n",
       "    pairing_count  expected_pairing_count  pairs_hist_count  \\\n",
       "37  6.0            4.0                     6.0                \n",
       "\n",
       "    expected_pairs_hist_count  affinity_count  expected_affinity_count  \n",
       "37  4.0                        55.0            56.0                     "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "invalid_group_pairings.query(\"group_pairing_id == @group_pair_id_tester\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_id</th>\n",
       "      <th>groupId</th>\n",
       "      <th>pairings</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>group_pairing_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>520</th>\n",
       "      <td>kDNiwKsweERmTmqqR</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST</td>\n",
       "      <td>[{'firstUserId': 'YnQ6fFpmuHvGe6QRy', 'firstUserName': 'Wen Si', 'secondUserId': '8qZ2ewe3dHAqXNZMY', 'secondUserName': 'Alea Huggins'}, {'firstUserId': 'QxRdbSHmSd3ZnfYep', 'firstUserName': 'Victoria Vann', 'secondUserId': 'QKt8uNR9BvNSGH3FT', 'secondUserName': 'Amy Hauenstein'}, {'firstUserId': 'CNrFoAHBw5pS5BLq3', 'firstUserName': 'George Geiger'}, {'firstUserId': 'iXESvat37kEqoogMo', 'firstUserName': 'Alyssa Ciancio'}, {'firstUserId': 'LPpJnfPBr5oDWfHfJ', 'firstUserName': 'Karen Rocha'}, {'firstUserId': 'h265JX9mRYKrj2jD2', 'firstUserName': 'Jenna Wilson'}]</td>\n",
       "      <td>2019-06-22 20:22:05.927</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST-kDNiwKsweERmTmqqR</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   _id            groupId  \\\n",
       "520  kDNiwKsweERmTmqqR  Pj9mwxXYPKxjPaxST   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    pairings  \\\n",
       "520  [{'firstUserId': 'YnQ6fFpmuHvGe6QRy', 'firstUserName': 'Wen Si', 'secondUserId': '8qZ2ewe3dHAqXNZMY', 'secondUserName': 'Alea Huggins'}, {'firstUserId': 'QxRdbSHmSd3ZnfYep', 'firstUserName': 'Victoria Vann', 'secondUserId': 'QKt8uNR9BvNSGH3FT', 'secondUserName': 'Amy Hauenstein'}, {'firstUserId': 'CNrFoAHBw5pS5BLq3', 'firstUserName': 'George Geiger'}, {'firstUserId': 'iXESvat37kEqoogMo', 'firstUserName': 'Alyssa Ciancio'}, {'firstUserId': 'LPpJnfPBr5oDWfHfJ', 'firstUserName': 'Karen Rocha'}, {'firstUserId': 'h265JX9mRYKrj2jD2', 'firstUserName': 'Jenna Wilson'}]   \n",
       "\n",
       "                  timestamp                     group_pairing_id  \n",
       "520 2019-06-22 20:22:05.927  Pj9mwxXYPKxjPaxST-kDNiwKsweERmTmqqR  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pairings.query(\"group_pairing_id == @group_pair_id_tester\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pairings.query(\"group_pairing_id ==  @group_pair_id_tester\").iloc[0]['pairings'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_id</th>\n",
       "      <th>name</th>\n",
       "      <th>userId</th>\n",
       "      <th>groupId</th>\n",
       "      <th>task</th>\n",
       "      <th>pairingId</th>\n",
       "      <th>group_pairing_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [_id, name, userId, groupId, task, pairingId, group_pairing_id]\n",
       "Index: []"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tasks_history.query(\"group_pairing_id == @group_pair_id_tester\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_id</th>\n",
       "      <th>name</th>\n",
       "      <th>userId</th>\n",
       "      <th>groupId</th>\n",
       "      <th>task</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>BvTQixBRziyaJSkAP</td>\n",
       "      <td>Jenna Wilson</td>\n",
       "      <td>h265JX9mRYKrj2jD2</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST</td>\n",
       "      <td>Does project management correlate with adult ed? If so, how?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>174</th>\n",
       "      <td>EDp9rDoETaB9MZPGu</td>\n",
       "      <td>Amy Hauenstein</td>\n",
       "      <td>QKt8uNR9BvNSGH3FT</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST</td>\n",
       "      <td>How can I help anyone?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>278</th>\n",
       "      <td>MGmrB7aup3Nbr4Whr</td>\n",
       "      <td>George Geiger</td>\n",
       "      <td>CNrFoAHBw5pS5BLq3</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST</td>\n",
       "      <td>What is an effective method to building milestones and measure development?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>347</th>\n",
       "      <td>RJnrfTGQBx3pC6Rwk</td>\n",
       "      <td>Alyssa Ciancio</td>\n",
       "      <td>iXESvat37kEqoogMo</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST</td>\n",
       "      <td>Useful tips and strategies for building out a new hire orientation.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>371</th>\n",
       "      <td>Sd6PPd5e9apinyguy</td>\n",
       "      <td>Karen Rocha</td>\n",
       "      <td>LPpJnfPBr5oDWfHfJ</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST</td>\n",
       "      <td>What are good mile markers in a training to \"quiz\" learners on the topics they have learned in the training to that point? Or should there even be \"quizzes\" in a training environment?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>487</th>\n",
       "      <td>bNi4oGKPfpY9Xqq3X</td>\n",
       "      <td>Alea Huggins</td>\n",
       "      <td>8qZ2ewe3dHAqXNZMY</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST</td>\n",
       "      <td>How do you determine the correct type of evaluation or metric to use for skills that are not easily measurable?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>501</th>\n",
       "      <td>cMWWoi8pRBojv4bqb</td>\n",
       "      <td>Kate Harrington-Rosen</td>\n",
       "      <td>7pDQQpubn4sdaQzHn</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST</td>\n",
       "      <td>What questions to ask in organizational assessments before implementing a workplace training, in order to assess organizational culture and readiness related to the topic.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>680</th>\n",
       "      <td>ovYSY5hasRjN7Ybq3</td>\n",
       "      <td>Victoria Vann</td>\n",
       "      <td>QxRdbSHmSd3ZnfYep</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST</td>\n",
       "      <td>I need help with deciding on a topic of interest and  creating a inclusive curriculum</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   _id                   name             userId  \\\n",
       "132  BvTQixBRziyaJSkAP  Jenna Wilson           h265JX9mRYKrj2jD2   \n",
       "174  EDp9rDoETaB9MZPGu  Amy Hauenstein         QKt8uNR9BvNSGH3FT   \n",
       "278  MGmrB7aup3Nbr4Whr  George Geiger          CNrFoAHBw5pS5BLq3   \n",
       "347  RJnrfTGQBx3pC6Rwk  Alyssa Ciancio         iXESvat37kEqoogMo   \n",
       "371  Sd6PPd5e9apinyguy  Karen Rocha            LPpJnfPBr5oDWfHfJ   \n",
       "487  bNi4oGKPfpY9Xqq3X  Alea Huggins           8qZ2ewe3dHAqXNZMY   \n",
       "501  cMWWoi8pRBojv4bqb  Kate Harrington-Rosen  7pDQQpubn4sdaQzHn   \n",
       "680  ovYSY5hasRjN7Ybq3  Victoria Vann          QxRdbSHmSd3ZnfYep   \n",
       "\n",
       "               groupId  \\\n",
       "132  Pj9mwxXYPKxjPaxST   \n",
       "174  Pj9mwxXYPKxjPaxST   \n",
       "278  Pj9mwxXYPKxjPaxST   \n",
       "347  Pj9mwxXYPKxjPaxST   \n",
       "371  Pj9mwxXYPKxjPaxST   \n",
       "487  Pj9mwxXYPKxjPaxST   \n",
       "501  Pj9mwxXYPKxjPaxST   \n",
       "680  Pj9mwxXYPKxjPaxST   \n",
       "\n",
       "                                                                                                                                                                                        task  \n",
       "132  Does project management correlate with adult ed? If so, how?                                                                                                                             \n",
       "174  How can I help anyone?                                                                                                                                                                   \n",
       "278  What is an effective method to building milestones and measure development?                                                                                                              \n",
       "347  Useful tips and strategies for building out a new hire orientation.                                                                                                                      \n",
       "371  What are good mile markers in a training to \"quiz\" learners on the topics they have learned in the training to that point? Or should there even be \"quizzes\" in a training environment?  \n",
       "487  How do you determine the correct type of evaluation or metric to use for skills that are not easily measurable?                                                                          \n",
       "501  What questions to ask in organizational assessments before implementing a workplace training, in order to assess organizational culture and readiness related to the topic.              \n",
       "680  I need help with deciding on a topic of interest and  creating a inclusive curriculum                                                                                                    "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tasks.query(\"groupId == 'Pj9mwxXYPKxjPaxST'\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_id</th>\n",
       "      <th>groupId</th>\n",
       "      <th>pairingId</th>\n",
       "      <th>firstUserId</th>\n",
       "      <th>firstUserName</th>\n",
       "      <th>firstUserRole</th>\n",
       "      <th>secondUserId</th>\n",
       "      <th>secondUserName</th>\n",
       "      <th>secondUserRole</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>group_pairing_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1761</th>\n",
       "      <td>Wr8EkwoRrZgThwx5f</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST</td>\n",
       "      <td>kDNiwKsweERmTmqqR</td>\n",
       "      <td>CNrFoAHBw5pS5BLq3</td>\n",
       "      <td>George Geiger</td>\n",
       "      <td>Student</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019-06-22 20:22:05.927</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST-kDNiwKsweERmTmqqR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3463</th>\n",
       "      <td>wbN9CEvesy7wTjwhj</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST</td>\n",
       "      <td>kDNiwKsweERmTmqqR</td>\n",
       "      <td>h265JX9mRYKrj2jD2</td>\n",
       "      <td>Jenna Wilson</td>\n",
       "      <td>Student</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019-06-22 20:22:05.927</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST-kDNiwKsweERmTmqqR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1347</th>\n",
       "      <td>N9kAfSswrtFTfZBQR</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST</td>\n",
       "      <td>kDNiwKsweERmTmqqR</td>\n",
       "      <td>LPpJnfPBr5oDWfHfJ</td>\n",
       "      <td>Karen Rocha</td>\n",
       "      <td>Student</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019-06-22 20:22:05.927</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST-kDNiwKsweERmTmqqR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1944</th>\n",
       "      <td>ZarnYRCWMyWnDFThW</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST</td>\n",
       "      <td>kDNiwKsweERmTmqqR</td>\n",
       "      <td>iXESvat37kEqoogMo</td>\n",
       "      <td>Alyssa Ciancio</td>\n",
       "      <td>Student</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019-06-22 20:22:05.927</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST-kDNiwKsweERmTmqqR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>660</th>\n",
       "      <td>CAQ7gEavFfkuXJwp9</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST</td>\n",
       "      <td>kDNiwKsweERmTmqqR</td>\n",
       "      <td>YnQ6fFpmuHvGe6QRy</td>\n",
       "      <td>Wen Si</td>\n",
       "      <td>Student</td>\n",
       "      <td>8qZ2ewe3dHAqXNZMY</td>\n",
       "      <td>Alea Huggins</td>\n",
       "      <td>Student</td>\n",
       "      <td>2019-06-22 20:22:05.927</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST-kDNiwKsweERmTmqqR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1516</th>\n",
       "      <td>QqbbvhPWHxhdCFsDE</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST</td>\n",
       "      <td>kDNiwKsweERmTmqqR</td>\n",
       "      <td>QxRdbSHmSd3ZnfYep</td>\n",
       "      <td>Victoria Vann</td>\n",
       "      <td>Student</td>\n",
       "      <td>QKt8uNR9BvNSGH3FT</td>\n",
       "      <td>Amy Hauenstein</td>\n",
       "      <td>Professor</td>\n",
       "      <td>2019-06-22 20:22:05.927</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST-kDNiwKsweERmTmqqR</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    _id            groupId          pairingId  \\\n",
       "1761  Wr8EkwoRrZgThwx5f  Pj9mwxXYPKxjPaxST  kDNiwKsweERmTmqqR   \n",
       "3463  wbN9CEvesy7wTjwhj  Pj9mwxXYPKxjPaxST  kDNiwKsweERmTmqqR   \n",
       "1347  N9kAfSswrtFTfZBQR  Pj9mwxXYPKxjPaxST  kDNiwKsweERmTmqqR   \n",
       "1944  ZarnYRCWMyWnDFThW  Pj9mwxXYPKxjPaxST  kDNiwKsweERmTmqqR   \n",
       "660   CAQ7gEavFfkuXJwp9  Pj9mwxXYPKxjPaxST  kDNiwKsweERmTmqqR   \n",
       "1516  QqbbvhPWHxhdCFsDE  Pj9mwxXYPKxjPaxST  kDNiwKsweERmTmqqR   \n",
       "\n",
       "            firstUserId   firstUserName firstUserRole       secondUserId  \\\n",
       "1761  CNrFoAHBw5pS5BLq3  George Geiger   Student       NaN                 \n",
       "3463  h265JX9mRYKrj2jD2  Jenna Wilson    Student       NaN                 \n",
       "1347  LPpJnfPBr5oDWfHfJ  Karen Rocha     Student       NaN                 \n",
       "1944  iXESvat37kEqoogMo  Alyssa Ciancio  Student       NaN                 \n",
       "660   YnQ6fFpmuHvGe6QRy  Wen Si          Student       8qZ2ewe3dHAqXNZMY   \n",
       "1516  QxRdbSHmSd3ZnfYep  Victoria Vann   Student       QKt8uNR9BvNSGH3FT   \n",
       "\n",
       "      secondUserName secondUserRole               timestamp  \\\n",
       "1761  NaN             NaN           2019-06-22 20:22:05.927   \n",
       "3463  NaN             NaN           2019-06-22 20:22:05.927   \n",
       "1347  NaN             NaN           2019-06-22 20:22:05.927   \n",
       "1944  NaN             NaN           2019-06-22 20:22:05.927   \n",
       "660   Alea Huggins    Student       2019-06-22 20:22:05.927   \n",
       "1516  Amy Hauenstein  Professor     2019-06-22 20:22:05.927   \n",
       "\n",
       "                         group_pairing_id  \n",
       "1761  Pj9mwxXYPKxjPaxST-kDNiwKsweERmTmqqR  \n",
       "3463  Pj9mwxXYPKxjPaxST-kDNiwKsweERmTmqqR  \n",
       "1347  Pj9mwxXYPKxjPaxST-kDNiwKsweERmTmqqR  \n",
       "1944  Pj9mwxXYPKxjPaxST-kDNiwKsweERmTmqqR  \n",
       "660   Pj9mwxXYPKxjPaxST-kDNiwKsweERmTmqqR  \n",
       "1516  Pj9mwxXYPKxjPaxST-kDNiwKsweERmTmqqR  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pairs_history.query(\"group_pairing_id == @group_pair_id_tester\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>group_pairing_id</th>\n",
       "      <th>group_id</th>\n",
       "      <th>pairing_id</th>\n",
       "      <th>conditions_failed</th>\n",
       "      <th>user_count</th>\n",
       "      <th>task_count</th>\n",
       "      <th>expected_task_count</th>\n",
       "      <th>pairing_count</th>\n",
       "      <th>expected_pairing_count</th>\n",
       "      <th>pairs_hist_count</th>\n",
       "      <th>expected_pairs_hist_count</th>\n",
       "      <th>affinity_count</th>\n",
       "      <th>expected_affinity_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Pj9mwxXYPKxjPaxST-MmfKpPtSv5TWDy8Jc</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST</td>\n",
       "      <td>MmfKpPtSv5TWDy8Jc</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>55.0</td>\n",
       "      <td>56.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>cToFEbgXcFbrKsSrj-9LidQoQiuSXdwJwoc</td>\n",
       "      <td>cToFEbgXcFbrKsSrj</td>\n",
       "      <td>9LidQoQiuSXdwJwoc</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>14.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>134.0</td>\n",
       "      <td>182.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>cToFEbgXcFbrKsSrj-Kevyvh5GoYBa6QhtS</td>\n",
       "      <td>cToFEbgXcFbrKsSrj</td>\n",
       "      <td>Kevyvh5GoYBa6QhtS</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>14.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>133.0</td>\n",
       "      <td>182.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt</td>\n",
       "      <td>cToFEbgXcFbrKsSrj</td>\n",
       "      <td>kTtxafCNsKoqsnhCt</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>14.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>131.0</td>\n",
       "      <td>182.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>fduEdDA8nk5ybcYze-9deZN5msXPWbRbaty</td>\n",
       "      <td>fduEdDA8nk5ybcYze</td>\n",
       "      <td>9deZN5msXPWbRbaty</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       group_pairing_id           group_id         pairing_id  \\\n",
       "21  Pj9mwxXYPKxjPaxST-MmfKpPtSv5TWDy8Jc  Pj9mwxXYPKxjPaxST  MmfKpPtSv5TWDy8Jc   \n",
       "22  cToFEbgXcFbrKsSrj-9LidQoQiuSXdwJwoc  cToFEbgXcFbrKsSrj  9LidQoQiuSXdwJwoc   \n",
       "23  cToFEbgXcFbrKsSrj-Kevyvh5GoYBa6QhtS  cToFEbgXcFbrKsSrj  Kevyvh5GoYBa6QhtS   \n",
       "24  cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt  cToFEbgXcFbrKsSrj  kTtxafCNsKoqsnhCt   \n",
       "25  fduEdDA8nk5ybcYze-9deZN5msXPWbRbaty  fduEdDA8nk5ybcYze  9deZN5msXPWbRbaty   \n",
       "\n",
       "   conditions_failed  user_count  task_count  expected_task_count  \\\n",
       "21  [1, 2, 3]         8.0         0.0         8.0                   \n",
       "22  [1, 2, 3]         14.0        7.0         14.0                  \n",
       "23  [1, 2, 3]         14.0        7.0         14.0                  \n",
       "24  [1, 2, 3]         14.0        7.0         14.0                  \n",
       "25  [1, 2, 3]         3.0         0.0         3.0                   \n",
       "\n",
       "    pairing_count  expected_pairing_count  pairs_hist_count  \\\n",
       "21  5.0            4.0                     5.0                \n",
       "22  8.0            7.0                     8.0                \n",
       "23  8.0            7.0                     8.0                \n",
       "24  8.0            7.0                     8.0                \n",
       "25  3.0            2.0                     3.0                \n",
       "\n",
       "    expected_pairs_hist_count  affinity_count  expected_affinity_count  \n",
       "21  4.0                        55.0            56.0                     \n",
       "22  7.0                        134.0           182.0                    \n",
       "23  7.0                        133.0           182.0                    \n",
       "24  7.0                        131.0           182.0                    \n",
       "25  2.0                        6.0             6.0                      "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "invalid_group_pairings[invalid_group_pairings.conditions_failed.apply(lambda x: '3' in x)].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>group_pairing_id</th>\n",
       "      <th>group_id</th>\n",
       "      <th>pairing_id</th>\n",
       "      <th>conditions_failed</th>\n",
       "      <th>user_count</th>\n",
       "      <th>task_count</th>\n",
       "      <th>expected_task_count</th>\n",
       "      <th>pairing_count</th>\n",
       "      <th>expected_pairing_count</th>\n",
       "      <th>pairs_hist_count</th>\n",
       "      <th>expected_pairs_hist_count</th>\n",
       "      <th>affinity_count</th>\n",
       "      <th>expected_affinity_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Pj9mwxXYPKxjPaxST-MmfKpPtSv5TWDy8Jc</td>\n",
       "      <td>Pj9mwxXYPKxjPaxST</td>\n",
       "      <td>MmfKpPtSv5TWDy8Jc</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>55.0</td>\n",
       "      <td>56.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>cToFEbgXcFbrKsSrj-9LidQoQiuSXdwJwoc</td>\n",
       "      <td>cToFEbgXcFbrKsSrj</td>\n",
       "      <td>9LidQoQiuSXdwJwoc</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>14.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>134.0</td>\n",
       "      <td>182.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>cToFEbgXcFbrKsSrj-Kevyvh5GoYBa6QhtS</td>\n",
       "      <td>cToFEbgXcFbrKsSrj</td>\n",
       "      <td>Kevyvh5GoYBa6QhtS</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>14.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>133.0</td>\n",
       "      <td>182.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt</td>\n",
       "      <td>cToFEbgXcFbrKsSrj</td>\n",
       "      <td>kTtxafCNsKoqsnhCt</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>14.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>131.0</td>\n",
       "      <td>182.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>fduEdDA8nk5ybcYze-9deZN5msXPWbRbaty</td>\n",
       "      <td>fduEdDA8nk5ybcYze</td>\n",
       "      <td>9deZN5msXPWbRbaty</td>\n",
       "      <td>[1, 2, 3]</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>238</th>\n",
       "      <td>FmHT4pnqrPXzCLCmE-AjouSu4Z9hjSLQ4sf</td>\n",
       "      <td>FmHT4pnqrPXzCLCmE</td>\n",
       "      <td>AjouSu4Z9hjSLQ4sf</td>\n",
       "      <td>[3]</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>239</th>\n",
       "      <td>FmHT4pnqrPXzCLCmE-HFQhCgbcKSPYAR45d</td>\n",
       "      <td>FmHT4pnqrPXzCLCmE</td>\n",
       "      <td>HFQhCgbcKSPYAR45d</td>\n",
       "      <td>[3]</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240</th>\n",
       "      <td>FmHT4pnqrPXzCLCmE-NQWmX9AJffCZDcBjx</td>\n",
       "      <td>FmHT4pnqrPXzCLCmE</td>\n",
       "      <td>NQWmX9AJffCZDcBjx</td>\n",
       "      <td>[3]</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>241</th>\n",
       "      <td>PAjekH9fE93A5jW2p-b7htbti6xyJHAKBX7</td>\n",
       "      <td>PAjekH9fE93A5jW2p</td>\n",
       "      <td>b7htbti6xyJHAKBX7</td>\n",
       "      <td>[3]</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>72.0</td>\n",
       "      <td>72.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>242</th>\n",
       "      <td>vwhT4maX84aXsfSGy-vMrakXnuz6huSJ7qd</td>\n",
       "      <td>vwhT4maX84aXsfSGy</td>\n",
       "      <td>vMrakXnuz6huSJ7qd</td>\n",
       "      <td>[3]</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>30.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>222 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                        group_pairing_id           group_id  \\\n",
       "21   Pj9mwxXYPKxjPaxST-MmfKpPtSv5TWDy8Jc  Pj9mwxXYPKxjPaxST   \n",
       "22   cToFEbgXcFbrKsSrj-9LidQoQiuSXdwJwoc  cToFEbgXcFbrKsSrj   \n",
       "23   cToFEbgXcFbrKsSrj-Kevyvh5GoYBa6QhtS  cToFEbgXcFbrKsSrj   \n",
       "24   cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt  cToFEbgXcFbrKsSrj   \n",
       "25   fduEdDA8nk5ybcYze-9deZN5msXPWbRbaty  fduEdDA8nk5ybcYze   \n",
       "..                                   ...                ...   \n",
       "238  FmHT4pnqrPXzCLCmE-AjouSu4Z9hjSLQ4sf  FmHT4pnqrPXzCLCmE   \n",
       "239  FmHT4pnqrPXzCLCmE-HFQhCgbcKSPYAR45d  FmHT4pnqrPXzCLCmE   \n",
       "240  FmHT4pnqrPXzCLCmE-NQWmX9AJffCZDcBjx  FmHT4pnqrPXzCLCmE   \n",
       "241  PAjekH9fE93A5jW2p-b7htbti6xyJHAKBX7  PAjekH9fE93A5jW2p   \n",
       "242  vwhT4maX84aXsfSGy-vMrakXnuz6huSJ7qd  vwhT4maX84aXsfSGy   \n",
       "\n",
       "            pairing_id conditions_failed  user_count  task_count  \\\n",
       "21   MmfKpPtSv5TWDy8Jc  [1, 2, 3]         8.0         0.0          \n",
       "22   9LidQoQiuSXdwJwoc  [1, 2, 3]         14.0        7.0          \n",
       "23   Kevyvh5GoYBa6QhtS  [1, 2, 3]         14.0        7.0          \n",
       "24   kTtxafCNsKoqsnhCt  [1, 2, 3]         14.0        7.0          \n",
       "25   9deZN5msXPWbRbaty  [1, 2, 3]         3.0         0.0          \n",
       "..                 ...        ...         ...         ...          \n",
       "238  AjouSu4Z9hjSLQ4sf  [3]               2.0         0.0          \n",
       "239  HFQhCgbcKSPYAR45d  [3]               2.0         0.0          \n",
       "240  NQWmX9AJffCZDcBjx  [3]               2.0         0.0          \n",
       "241  b7htbti6xyJHAKBX7  [3]               9.0         0.0          \n",
       "242  vMrakXnuz6huSJ7qd  [3]               6.0         0.0          \n",
       "\n",
       "     expected_task_count  pairing_count  expected_pairing_count  \\\n",
       "21   8.0                  5.0            4.0                      \n",
       "22   14.0                 8.0            7.0                      \n",
       "23   14.0                 8.0            7.0                      \n",
       "24   14.0                 8.0            7.0                      \n",
       "25   3.0                  3.0            2.0                      \n",
       "..   ...                  ...            ...                      \n",
       "238  2.0                  1.0            1.0                      \n",
       "239  2.0                  1.0            1.0                      \n",
       "240  2.0                  1.0            1.0                      \n",
       "241  9.0                  5.0            5.0                      \n",
       "242  6.0                  3.0            3.0                      \n",
       "\n",
       "     pairs_hist_count  expected_pairs_hist_count  affinity_count  \\\n",
       "21   5.0               4.0                        55.0             \n",
       "22   8.0               7.0                        134.0            \n",
       "23   8.0               7.0                        133.0            \n",
       "24   8.0               7.0                        131.0            \n",
       "25   3.0               2.0                        6.0              \n",
       "..   ...               ...                        ...              \n",
       "238  1.0               1.0                        1.0              \n",
       "239  1.0               1.0                        1.0              \n",
       "240  1.0               1.0                        1.0              \n",
       "241  5.0               5.0                        72.0             \n",
       "242  3.0               3.0                        21.0             \n",
       "\n",
       "     expected_affinity_count  \n",
       "21   56.0                     \n",
       "22   182.0                    \n",
       "23   182.0                    \n",
       "24   182.0                    \n",
       "25   6.0                      \n",
       "..   ...                      \n",
       "238  2.0                      \n",
       "239  2.0                      \n",
       "240  2.0                      \n",
       "241  72.0                     \n",
       "242  30.0                     \n",
       "\n",
       "[222 rows x 13 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "invalid_group_pairings[invalid_group_pairings.conditions_failed.apply(lambda x: '3' in x)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_id</th>\n",
       "      <th>name</th>\n",
       "      <th>userId</th>\n",
       "      <th>groupId</th>\n",
       "      <th>task</th>\n",
       "      <th>pairingId</th>\n",
       "      <th>group_pairing_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>156</th>\n",
       "      <td>3rkteqrAXyWqBoeCZ</td>\n",
       "      <td>Akshay Iyer</td>\n",
       "      <td>4qc4JR2do9vEYuzSo</td>\n",
       "      <td>cToFEbgXcFbrKsSrj</td>\n",
       "      <td>1) Understanding phase-retrieval method for image reconstruction. 2) Testing my MCR webtools.</td>\n",
       "      <td>kTtxafCNsKoqsnhCt</td>\n",
       "      <td>cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>923</th>\n",
       "      <td>BT9s4mgSvnbnFzd95</td>\n",
       "      <td>Wei Chen</td>\n",
       "      <td>NEdS2iCrcCqqoQ7hk</td>\n",
       "      <td>cToFEbgXcFbrKsSrj</td>\n",
       "      <td>merge references, format section titles, cross reference figures  in word document</td>\n",
       "      <td>kTtxafCNsKoqsnhCt</td>\n",
       "      <td>cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>959</th>\n",
       "      <td>BtLznF3BPX3BXvHrz</td>\n",
       "      <td>Jun Lu</td>\n",
       "      <td>orxqD8rRRPzBGjMPN</td>\n",
       "      <td>cToFEbgXcFbrKsSrj</td>\n",
       "      <td>Bayesian calibration code writing</td>\n",
       "      <td>kTtxafCNsKoqsnhCt</td>\n",
       "      <td>cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1898</th>\n",
       "      <td>MoCkatmpwoXrDbBzG</td>\n",
       "      <td>Umar Farooq Ghumman</td>\n",
       "      <td>RxToKSgo8zMdNioPy</td>\n",
       "      <td>cToFEbgXcFbrKsSrj</td>\n",
       "      <td>Understanding effect of padding on reconstruction using 1d SDF</td>\n",
       "      <td>kTtxafCNsKoqsnhCt</td>\n",
       "      <td>cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2249</th>\n",
       "      <td>R33jtkfATLzKdiuNW</td>\n",
       "      <td>Jun Hu</td>\n",
       "      <td>NoRwBcWpmWcRfQjTN</td>\n",
       "      <td>cToFEbgXcFbrKsSrj</td>\n",
       "      <td>Run matlab code in linux system</td>\n",
       "      <td>kTtxafCNsKoqsnhCt</td>\n",
       "      <td>cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3686</th>\n",
       "      <td>i3pFKHWRhCECLtJCN</td>\n",
       "      <td>Siyu Tao</td>\n",
       "      <td>j4uFBiRu2NhS3M5cf</td>\n",
       "      <td>cToFEbgXcFbrKsSrj</td>\n",
       "      <td>General chats; Object visualization &amp; Articulation</td>\n",
       "      <td>kTtxafCNsKoqsnhCt</td>\n",
       "      <td>cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4222</th>\n",
       "      <td>oWNHwWE6hZTraqJMw</td>\n",
       "      <td>Yichi Zhang</td>\n",
       "      <td>EEorexkFoARrSqCme</td>\n",
       "      <td>cToFEbgXcFbrKsSrj</td>\n",
       "      <td>Linux Server Operation</td>\n",
       "      <td>kTtxafCNsKoqsnhCt</td>\n",
       "      <td>cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    _id                 name             userId  \\\n",
       "156   3rkteqrAXyWqBoeCZ  Akshay Iyer          4qc4JR2do9vEYuzSo   \n",
       "923   BT9s4mgSvnbnFzd95  Wei Chen             NEdS2iCrcCqqoQ7hk   \n",
       "959   BtLznF3BPX3BXvHrz  Jun Lu               orxqD8rRRPzBGjMPN   \n",
       "1898  MoCkatmpwoXrDbBzG  Umar Farooq Ghumman  RxToKSgo8zMdNioPy   \n",
       "2249  R33jtkfATLzKdiuNW  Jun Hu               NoRwBcWpmWcRfQjTN   \n",
       "3686  i3pFKHWRhCECLtJCN  Siyu Tao             j4uFBiRu2NhS3M5cf   \n",
       "4222  oWNHwWE6hZTraqJMw  Yichi Zhang          EEorexkFoARrSqCme   \n",
       "\n",
       "                groupId  \\\n",
       "156   cToFEbgXcFbrKsSrj   \n",
       "923   cToFEbgXcFbrKsSrj   \n",
       "959   cToFEbgXcFbrKsSrj   \n",
       "1898  cToFEbgXcFbrKsSrj   \n",
       "2249  cToFEbgXcFbrKsSrj   \n",
       "3686  cToFEbgXcFbrKsSrj   \n",
       "4222  cToFEbgXcFbrKsSrj   \n",
       "\n",
       "                                                                                               task  \\\n",
       "156   1) Understanding phase-retrieval method for image reconstruction. 2) Testing my MCR webtools.   \n",
       "923   merge references, format section titles, cross reference figures  in word document              \n",
       "959   Bayesian calibration code writing                                                               \n",
       "1898  Understanding effect of padding on reconstruction using 1d SDF                                  \n",
       "2249  Run matlab code in linux system                                                                 \n",
       "3686  General chats; Object visualization & Articulation                                              \n",
       "4222  Linux Server Operation                                                                          \n",
       "\n",
       "              pairingId                     group_pairing_id  \n",
       "156   kTtxafCNsKoqsnhCt  cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt  \n",
       "923   kTtxafCNsKoqsnhCt  cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt  \n",
       "959   kTtxafCNsKoqsnhCt  cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt  \n",
       "1898  kTtxafCNsKoqsnhCt  cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt  \n",
       "2249  kTtxafCNsKoqsnhCt  cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt  \n",
       "3686  kTtxafCNsKoqsnhCt  cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt  \n",
       "4222  kTtxafCNsKoqsnhCt  cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tasks_history.query(\"group_pairing_id == 'cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt'\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_id</th>\n",
       "      <th>groupId</th>\n",
       "      <th>pairingId</th>\n",
       "      <th>firstUserId</th>\n",
       "      <th>firstUserName</th>\n",
       "      <th>firstUserRole</th>\n",
       "      <th>secondUserId</th>\n",
       "      <th>secondUserName</th>\n",
       "      <th>secondUserRole</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>group_pairing_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>838</th>\n",
       "      <td>EqSokGjjsWEpDxu9E</td>\n",
       "      <td>cToFEbgXcFbrKsSrj</td>\n",
       "      <td>kTtxafCNsKoqsnhCt</td>\n",
       "      <td>fb6wtYuZepzf4x3xP</td>\n",
       "      <td>Youyi Bi</td>\n",
       "      <td>Post Doc</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018-02-19 18:14:17.855</td>\n",
       "      <td>cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3165</th>\n",
       "      <td>rpthZ8fhCjpRabGCq</td>\n",
       "      <td>cToFEbgXcFbrKsSrj</td>\n",
       "      <td>kTtxafCNsKoqsnhCt</td>\n",
       "      <td>6qXcX9zenXnu2nkrK</td>\n",
       "      <td>Anton  van  beek</td>\n",
       "      <td>Graduate Student</td>\n",
       "      <td>C7DLymK5MSiXRgfto</td>\n",
       "      <td>Ramin Bostanabad</td>\n",
       "      <td>Graduate Student</td>\n",
       "      <td>2018-02-19 18:14:17.855</td>\n",
       "      <td>cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>348</th>\n",
       "      <td>7NABRygap6iwCsH7u</td>\n",
       "      <td>cToFEbgXcFbrKsSrj</td>\n",
       "      <td>kTtxafCNsKoqsnhCt</td>\n",
       "      <td>4qc4JR2do9vEYuzSo</td>\n",
       "      <td>Akshay Iyer</td>\n",
       "      <td>Graduate Student</td>\n",
       "      <td>wmxEuJ8CZBQExYDoq</td>\n",
       "      <td>Yu-Chin Chan</td>\n",
       "      <td>Graduate Student</td>\n",
       "      <td>2018-02-19 18:14:17.855</td>\n",
       "      <td>cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3050</th>\n",
       "      <td>pvi6YA3fTKz9oqjkz</td>\n",
       "      <td>cToFEbgXcFbrKsSrj</td>\n",
       "      <td>kTtxafCNsKoqsnhCt</td>\n",
       "      <td>RxToKSgo8zMdNioPy</td>\n",
       "      <td>Umar Farooq Ghumman</td>\n",
       "      <td>Graduate Student</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018-02-19 18:14:17.855</td>\n",
       "      <td>cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>371</th>\n",
       "      <td>7heZSxp5yfvjN8N2c</td>\n",
       "      <td>cToFEbgXcFbrKsSrj</td>\n",
       "      <td>kTtxafCNsKoqsnhCt</td>\n",
       "      <td>NEdS2iCrcCqqoQ7hk</td>\n",
       "      <td>Wei Chen</td>\n",
       "      <td>Professor</td>\n",
       "      <td>S5aKovd9M7vhJoBJu</td>\n",
       "      <td>Jian Xie</td>\n",
       "      <td>Graduate Student</td>\n",
       "      <td>2018-02-19 18:14:17.855</td>\n",
       "      <td>cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1153</th>\n",
       "      <td>K8tj4wberLQwtFH8g</td>\n",
       "      <td>cToFEbgXcFbrKsSrj</td>\n",
       "      <td>kTtxafCNsKoqsnhCt</td>\n",
       "      <td>orxqD8rRRPzBGjMPN</td>\n",
       "      <td>Jun Lu</td>\n",
       "      <td>Graduate Student</td>\n",
       "      <td>HADAAreJ9u8aW37Gj</td>\n",
       "      <td>Xiaolin Li</td>\n",
       "      <td>Graduate Student</td>\n",
       "      <td>2018-02-19 18:14:17.855</td>\n",
       "      <td>cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2544</th>\n",
       "      <td>hi4E9kaqC5XCEKWj2</td>\n",
       "      <td>cToFEbgXcFbrKsSrj</td>\n",
       "      <td>kTtxafCNsKoqsnhCt</td>\n",
       "      <td>NoRwBcWpmWcRfQjTN</td>\n",
       "      <td>Jun Hu</td>\n",
       "      <td>Graduate Student</td>\n",
       "      <td>j4uFBiRu2NhS3M5cf</td>\n",
       "      <td>Siyu Tao</td>\n",
       "      <td>Graduate Student</td>\n",
       "      <td>2018-02-19 18:14:17.855</td>\n",
       "      <td>cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1940</th>\n",
       "      <td>ZXFGvC2Q88gFmm2Jm</td>\n",
       "      <td>cToFEbgXcFbrKsSrj</td>\n",
       "      <td>kTtxafCNsKoqsnhCt</td>\n",
       "      <td>EEorexkFoARrSqCme</td>\n",
       "      <td>Yichi Zhang</td>\n",
       "      <td>Graduate Student</td>\n",
       "      <td>WwRReXiB43ys4YAHM</td>\n",
       "      <td>Tianyu Huang</td>\n",
       "      <td>Graduate Student</td>\n",
       "      <td>2018-02-19 18:14:17.855</td>\n",
       "      <td>cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    _id            groupId          pairingId  \\\n",
       "838   EqSokGjjsWEpDxu9E  cToFEbgXcFbrKsSrj  kTtxafCNsKoqsnhCt   \n",
       "3165  rpthZ8fhCjpRabGCq  cToFEbgXcFbrKsSrj  kTtxafCNsKoqsnhCt   \n",
       "348   7NABRygap6iwCsH7u  cToFEbgXcFbrKsSrj  kTtxafCNsKoqsnhCt   \n",
       "3050  pvi6YA3fTKz9oqjkz  cToFEbgXcFbrKsSrj  kTtxafCNsKoqsnhCt   \n",
       "371   7heZSxp5yfvjN8N2c  cToFEbgXcFbrKsSrj  kTtxafCNsKoqsnhCt   \n",
       "1153  K8tj4wberLQwtFH8g  cToFEbgXcFbrKsSrj  kTtxafCNsKoqsnhCt   \n",
       "2544  hi4E9kaqC5XCEKWj2  cToFEbgXcFbrKsSrj  kTtxafCNsKoqsnhCt   \n",
       "1940  ZXFGvC2Q88gFmm2Jm  cToFEbgXcFbrKsSrj  kTtxafCNsKoqsnhCt   \n",
       "\n",
       "            firstUserId        firstUserName     firstUserRole  \\\n",
       "838   fb6wtYuZepzf4x3xP  Youyi Bi             Post Doc           \n",
       "3165  6qXcX9zenXnu2nkrK  Anton  van  beek     Graduate Student   \n",
       "348   4qc4JR2do9vEYuzSo  Akshay Iyer          Graduate Student   \n",
       "3050  RxToKSgo8zMdNioPy  Umar Farooq Ghumman  Graduate Student   \n",
       "371   NEdS2iCrcCqqoQ7hk  Wei Chen             Professor          \n",
       "1153  orxqD8rRRPzBGjMPN  Jun Lu               Graduate Student   \n",
       "2544  NoRwBcWpmWcRfQjTN  Jun Hu               Graduate Student   \n",
       "1940  EEorexkFoARrSqCme  Yichi Zhang          Graduate Student   \n",
       "\n",
       "           secondUserId    secondUserName    secondUserRole  \\\n",
       "838   NaN                NaN               NaN                \n",
       "3165  C7DLymK5MSiXRgfto  Ramin Bostanabad  Graduate Student   \n",
       "348   wmxEuJ8CZBQExYDoq  Yu-Chin Chan      Graduate Student   \n",
       "3050  NaN                NaN               NaN                \n",
       "371   S5aKovd9M7vhJoBJu  Jian Xie          Graduate Student   \n",
       "1153  HADAAreJ9u8aW37Gj  Xiaolin Li        Graduate Student   \n",
       "2544  j4uFBiRu2NhS3M5cf  Siyu Tao          Graduate Student   \n",
       "1940  WwRReXiB43ys4YAHM  Tianyu Huang      Graduate Student   \n",
       "\n",
       "                   timestamp                     group_pairing_id  \n",
       "838  2018-02-19 18:14:17.855  cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt  \n",
       "3165 2018-02-19 18:14:17.855  cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt  \n",
       "348  2018-02-19 18:14:17.855  cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt  \n",
       "3050 2018-02-19 18:14:17.855  cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt  \n",
       "371  2018-02-19 18:14:17.855  cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt  \n",
       "1153 2018-02-19 18:14:17.855  cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt  \n",
       "2544 2018-02-19 18:14:17.855  cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt  \n",
       "1940 2018-02-19 18:14:17.855  cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pairs_history.query(\"group_pairing_id == 'cToFEbgXcFbrKsSrj-kTtxafCNsKoqsnhCt'\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Filter out invalid pairing sessions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_invalid_sessions(df, df_name, exclusion_list):\n",
    "    \"\"\"\n",
    "    Removes invalid group-pairing session ids from dataframe.\n",
    "    \n",
    "    Input:\n",
    "        df (pandas dataframe): dataframe to remove group-pairing sessions from.\n",
    "        df_name (string): name of data frame.\n",
    "        exclusion_list (list of strings): group_pairing_ids to remove from df.\n",
    "        \n",
    "    Output:\n",
    "        (pandas dataframe): cleaned dataframe\n",
    "    \"\"\"\n",
    "    # track size for printing\n",
    "    orig_size = len(df)\n",
    "    new_size = 0\n",
    "    \n",
    "    # clean data\n",
    "    df = df[~df['group_pairing_id'].isin(exclusion_list)]\n",
    "    df.reset_index(drop=True, inplace=True)\n",
    "    new_size = len(df)\n",
    "    \n",
    "    # print change in size\n",
    "    print('{} Cleaning \\nOrig Size: {} ==> New Size: {}'.format(df_name, orig_size, new_size), end='\\n\\n')\n",
    "    \n",
    "    # return cleaned df\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "invalid_group_pairing_ids_list = list(invalid_group_pairings['group_pairing_id'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tasks_history Cleaning \n",
      "Orig Size: 6138 ==> New Size: 5579\n",
      "\n",
      "pairings Cleaning \n",
      "Orig Size: 800 ==> New Size: 557\n",
      "\n",
      "pairs_history Cleaning \n",
      "Orig Size: 4129 ==> New Size: 2944\n",
      "\n",
      "affinities_history Cleaning \n",
      "Orig Size: 71581 ==> New Size: 53267\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tasks_history = remove_invalid_sessions(tasks_history, 'tasks_history', invalid_group_pairing_ids_list)\n",
    "pairings = remove_invalid_sessions(pairings, 'pairings', invalid_group_pairing_ids_list)\n",
    "pairs_history = remove_invalid_sessions(pairs_history, 'pairs_history', invalid_group_pairing_ids_list)\n",
    "affinities_history = remove_invalid_sessions(affinities_history, 'affinities_history', invalid_group_pairing_ids_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add unique sequence number to each pairing session for each group\n",
    "For example, first session for DTR is 1, second is 2, etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>groupId</th>\n",
       "      <th>pairingId</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>group_pairing_id</th>\n",
       "      <th>pairing_session_index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2rFoGTfRa9LFdpQNA</td>\n",
       "      <td>A6d3rQwrRZHEz4qHu</td>\n",
       "      <td>2017-08-22 17:19:36.847</td>\n",
       "      <td>2rFoGTfRa9LFdpQNA-A6d3rQwrRZHEz4qHu</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2rFoGTfRa9LFdpQNA</td>\n",
       "      <td>pNFhXhotBwQ6Z79Md</td>\n",
       "      <td>2017-08-22 21:54:05.882</td>\n",
       "      <td>2rFoGTfRa9LFdpQNA-pNFhXhotBwQ6Z79Md</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2rFoGTfRa9LFdpQNA</td>\n",
       "      <td>eSXY7BuRX3ZhXX627</td>\n",
       "      <td>2017-10-24 20:56:43.001</td>\n",
       "      <td>2rFoGTfRa9LFdpQNA-eSXY7BuRX3ZhXX627</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2rFoGTfRa9LFdpQNA</td>\n",
       "      <td>SwhcfsdjNCZcyzx3t</td>\n",
       "      <td>2017-11-28 21:48:06.568</td>\n",
       "      <td>2rFoGTfRa9LFdpQNA-SwhcfsdjNCZcyzx3t</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2rFoGTfRa9LFdpQNA</td>\n",
       "      <td>SpiKfuqCoEZRLfDNK</td>\n",
       "      <td>2018-01-16 21:42:19.584</td>\n",
       "      <td>2rFoGTfRa9LFdpQNA-SpiKfuqCoEZRLfDNK</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             groupId          pairingId               timestamp  \\\n",
       "0  2rFoGTfRa9LFdpQNA  A6d3rQwrRZHEz4qHu 2017-08-22 17:19:36.847   \n",
       "1  2rFoGTfRa9LFdpQNA  pNFhXhotBwQ6Z79Md 2017-08-22 21:54:05.882   \n",
       "2  2rFoGTfRa9LFdpQNA  eSXY7BuRX3ZhXX627 2017-10-24 20:56:43.001   \n",
       "3  2rFoGTfRa9LFdpQNA  SwhcfsdjNCZcyzx3t 2017-11-28 21:48:06.568   \n",
       "4  2rFoGTfRa9LFdpQNA  SpiKfuqCoEZRLfDNK 2018-01-16 21:42:19.584   \n",
       "\n",
       "                      group_pairing_id  pairing_session_index  \n",
       "0  2rFoGTfRa9LFdpQNA-A6d3rQwrRZHEz4qHu  1                      \n",
       "1  2rFoGTfRa9LFdpQNA-pNFhXhotBwQ6Z79Md  2                      \n",
       "2  2rFoGTfRa9LFdpQNA-eSXY7BuRX3ZhXX627  3                      \n",
       "3  2rFoGTfRa9LFdpQNA-SwhcfsdjNCZcyzx3t  4                      \n",
       "4  2rFoGTfRa9LFdpQNA-SpiKfuqCoEZRLfDNK  5                      "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extract sorted pairing ids\n",
    "pairings_id_df = pairs_history[['groupId', 'pairingId', 'timestamp']].drop_duplicates(subset=['groupId', 'pairingId'], keep='first').sort_values(['groupId', 'timestamp']).reset_index(drop=True)\n",
    "pairings_id_df['group_pairing_id'] = pairings_id_df['groupId'] + '-' + pairings_id_df['pairingId']\n",
    "pairings_id_df['pairing_session_index'] = 0\n",
    "\n",
    "# add index label for each\n",
    "for groupId in pairings_id_df['groupId'].unique():\n",
    "    n_row = len(pairings_id_df[pairings_id_df['groupId'] == groupId])\n",
    "    pairings_id_df.loc[pairings_id_df.groupId == groupId, 'pairing_session_index'] = range(1, n_row + 1)\n",
    "\n",
    "# show top 5 frows\n",
    "pairings_id_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add pairing index to earlier data frames\n",
    "tasks_history = tasks_history.merge(pairings_id_df[['group_pairing_id', 'pairing_session_index']], on=['group_pairing_id'])\n",
    "pairings = pairings.merge(pairings_id_df[['group_pairing_id', 'pairing_session_index']], on=['group_pairing_id'])\n",
    "pairs_history = pairs_history.merge(pairings_id_df[['group_pairing_id', 'pairing_session_index']], on=['group_pairing_id'])\n",
    "affinities_history = affinities_history.merge(pairings_id_df[['group_pairing_id', 'pairing_session_index']], on=['group_pairing_id'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Summary Stats"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Groups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of unique groups: 129\n"
     ]
    }
   ],
   "source": [
    "# number of unique groups\n",
    "print('Number of unique groups: {}'.format(len(groups)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of pairing sessions: 557\n"
     ]
    }
   ],
   "source": [
    "# total number of pairing sessions\n",
    "print('Number of pairing sessions: {}'.format(len(pairs_history['group_pairing_id'].unique())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of unique groups who have participated in at least one session: 43\n"
     ]
    }
   ],
   "source": [
    "# TODO: this stat doesn't look right...check to see if the data is saving correctly in meteor\n",
    "# groups that have participated in a pairing session\n",
    "print('Number of unique groups who have participated in at least one session: {}'.format(len(pairs_history['groupId'].unique())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of pairing sessions by group\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>groupId</th>\n",
       "      <th>groupName</th>\n",
       "      <th>description</th>\n",
       "      <th>creatorName</th>\n",
       "      <th>pairingCount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF</td>\n",
       "      <td>Delta Lab</td>\n",
       "      <td>an interdisciplinary research group and design studio at Northwestern University</td>\n",
       "      <td>Haoqi Zhang</td>\n",
       "      <td>213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>sM3z5FkZfsABqcj3g</td>\n",
       "      <td>Design, Technology, and Research</td>\n",
       "      <td>Design, Technology, and Research program at Northwestern University</td>\n",
       "      <td>Haoqi Zhang</td>\n",
       "      <td>126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>BibLRuKtNNv7QEDqb</td>\n",
       "      <td>KIXLAB</td>\n",
       "      <td>The KAIST Interaction Lab</td>\n",
       "      <td>Juho Kim</td>\n",
       "      <td>84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>cToFEbgXcFbrKsSrj</td>\n",
       "      <td>IDEAL</td>\n",
       "      <td>Wei Chen's research lab</td>\n",
       "      <td>Wei Chen</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>PeACz8rabRn9BJRJc</td>\n",
       "      <td>AY17 MSC 538-0</td>\n",
       "      <td>Graduate Class: Workplace Learning &amp; Communities of Practice 2017</td>\n",
       "      <td>Amy Hauenstein</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>GWWtfSL8J4ZfWR2Fw</td>\n",
       "      <td>Research Discussion</td>\n",
       "      <td>research</td>\n",
       "      <td>Janet Huang</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2sikqRCzCtNPfynTq</td>\n",
       "      <td>Design of Playful Learning Environments</td>\n",
       "      <td>Grad seminar in the Learning Sciences</td>\n",
       "      <td>Nell O'Rourke</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>qPnf2DHHihugATnxD</td>\n",
       "      <td>Segal Design Cluster</td>\n",
       "      <td>an intellectual community for design faculty and doctoral students at Northwestern</td>\n",
       "      <td>Haoqi Zhang</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2rFoGTfRa9LFdpQNA</td>\n",
       "      <td>DAPS Seminar 2017-2018</td>\n",
       "      <td>Making pairs for DAPS seminar student workshopping</td>\n",
       "      <td>Katie Cheng</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>eKPuSBq5XSLZmMy4g</td>\n",
       "      <td>SDG+UP Pair Research</td>\n",
       "      <td>Software Design Group and Usable Programming group</td>\n",
       "      <td>Rob Miller</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              groupId                                groupName  \\\n",
       "6   9mdkMmj4pY8Q2TwqF  Delta Lab                                 \n",
       "37  sM3z5FkZfsABqcj3g  Design, Technology, and Research          \n",
       "8   BibLRuKtNNv7QEDqb  KIXLAB                                    \n",
       "27  cToFEbgXcFbrKsSrj  IDEAL                                     \n",
       "18  PeACz8rabRn9BJRJc  AY17 MSC 538-0                            \n",
       "11  GWWtfSL8J4ZfWR2Fw  Research Discussion                       \n",
       "1   2sikqRCzCtNPfynTq  Design of Playful Learning Environments   \n",
       "36  qPnf2DHHihugATnxD  Segal Design Cluster                      \n",
       "0   2rFoGTfRa9LFdpQNA  DAPS Seminar 2017-2018                    \n",
       "29  eKPuSBq5XSLZmMy4g  SDG+UP Pair Research                      \n",
       "\n",
       "                                                                           description  \\\n",
       "6   an interdisciplinary research group and design studio at Northwestern University     \n",
       "37  Design, Technology, and Research program at Northwestern University                  \n",
       "8   The KAIST Interaction Lab                                                            \n",
       "27  Wei Chen's research lab                                                              \n",
       "18  Graduate Class: Workplace Learning & Communities of Practice 2017                    \n",
       "11  research                                                                             \n",
       "1   Grad seminar in the Learning Sciences                                                \n",
       "36  an intellectual community for design faculty and doctoral students at Northwestern   \n",
       "0   Making pairs for DAPS seminar student workshopping                                   \n",
       "29  Software Design Group and Usable Programming group                                   \n",
       "\n",
       "       creatorName  pairingCount  \n",
       "6   Haoqi Zhang     213           \n",
       "37  Haoqi Zhang     126           \n",
       "8   Juho Kim        84            \n",
       "27  Wei Chen        25            \n",
       "18  Amy Hauenstein  15            \n",
       "11  Janet Huang     13            \n",
       "1   Nell O'Rourke   7             \n",
       "36  Haoqi Zhang     6             \n",
       "0   Katie Cheng     5             \n",
       "29  Rob Miller      5             "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# groups by number of pairing sessions\n",
    "pairing_count_bygroup = pairs_history.drop_duplicates('group_pairing_id').groupby('groupId')['groupId'].size().to_frame(name = 'pairingCount').reset_index()\n",
    "pairing_count_bygroup_named = pairing_count_bygroup.merge(groups, left_on='groupId', right_on='_id')[['groupId', 'groupName', 'description', 'creatorName', 'pairingCount']]\n",
    "pairing_count_bygroup_named.sort_values('pairingCount', ascending=False, inplace=True)\n",
    "\n",
    "print('Number of pairing sessions by group')\n",
    "pairing_count_bygroup_named.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    43.000000 \n",
       "mean     12.953488 \n",
       "std      38.538758 \n",
       "min      1.000000  \n",
       "25%      1.000000  \n",
       "50%      2.000000  \n",
       "75%      4.500000  \n",
       "max      213.000000\n",
       "Name: pairingCount, dtype: float64"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# distribution of pairings per group\n",
    "pairing_count_bygroup_named['pairingCount'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# separate out groups and group creators for labeling\n",
    "temp_groups = groups[['_id', 'groupName', 'description', 'creatorName']].rename(columns={'_id': 'groupId'})                      \n",
    "all_group_counts = pairing_count_bygroup_named.merge(temp_groups,\n",
    "                                                     how='outer',\n",
    "                                                     on=['groupId', 'groupName', 'description', 'creatorName'])\n",
    "all_group_counts.fillna(0, inplace=True)\n",
    "all_group_counts.rename(columns={\n",
    "    'groupId': 'Unique Identifier',\n",
    "    'groupName': 'Group Name',\n",
    "    'description': 'Group Description',\n",
    "    'creatorName': 'Group Creator',\n",
    "    'pairingCount': 'Number of Pairing Sessions Held'\n",
    "}, inplace=True)\n",
    "\n",
    "\n",
    "# show groups and write to csv\n",
    "all_group_counts\n",
    "all_group_counts.to_csv('./documents/groups_pairing-counts.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Individuals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Unique users: 1322\n"
     ]
    }
   ],
   "source": [
    "# number of unique accounts\n",
    "print('Number of Unique users: {}'.format(len(users)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Unique Users who have participated in pairings: 475\n"
     ]
    }
   ],
   "source": [
    "# TODO: this stat doesn't look right...check to see if the data is saving correctly in meteor\n",
    "# number of unique users who have paricipated in at least one session\n",
    "print('Number of Unique Users who have participated in pairings: {}'.format(len(set(pairs_history['firstUserId'].unique()).union(set(pairs_history['secondUserId'].unique())))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "557"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pairs_history[\"group_pairing_id\"].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of pairings: 2944\n"
     ]
    }
   ],
   "source": [
    "# get total number of pairings\n",
    "print('Total number of pairings: {}'.format(len(pairs_history)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of pairing sessions by user\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userId</th>\n",
       "      <th>profile</th>\n",
       "      <th>pairingCount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>234</th>\n",
       "      <td>Z68eenXDt84czPkmN</td>\n",
       "      <td>{'fullName': 'Ryan Louie', 'avatar': 'https://farm9.staticflickr.com/8281/7630259612_1e199cd81a.jpg'}</td>\n",
       "      <td>207</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>448</th>\n",
       "      <td>zmwK4tJHtwLw8pLRC</td>\n",
       "      <td>{'fullName': 'Garrett', 'avatar': 'https://cdn2.scratch.mit.edu/get_image/gallery/2090239_170x100.png'}</td>\n",
       "      <td>175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>248</th>\n",
       "      <td>aNdSTecskgeAm2St5</td>\n",
       "      <td>{'fullName': 'Leesha', 'avatar': 'https://imgbin.com/png/wKHccVQ4/pac-man-cherry-post-it-note-t-shirt-sticker-png'}</td>\n",
       "      <td>174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>344</th>\n",
       "      <td>mdhFQ6PNiAhfP7ce2</td>\n",
       "      <td>{'fullName': 'Kapil Garg', 'avatar': 'https://www.kgarg.com/img/profile.jpg'}</td>\n",
       "      <td>174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>331</th>\n",
       "      <td>jkwZFtFa69FijNWyY</td>\n",
       "      <td>{'fullName': 'Jamie Gorson', 'avatar': 'http://tsb.northwestern.edu/img/gorson.jpg'}</td>\n",
       "      <td>161</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                userId  \\\n",
       "234  Z68eenXDt84czPkmN   \n",
       "448  zmwK4tJHtwLw8pLRC   \n",
       "248  aNdSTecskgeAm2St5   \n",
       "344  mdhFQ6PNiAhfP7ce2   \n",
       "331  jkwZFtFa69FijNWyY   \n",
       "\n",
       "                                                                                                                 profile  \\\n",
       "234  {'fullName': 'Ryan Louie', 'avatar': 'https://farm9.staticflickr.com/8281/7630259612_1e199cd81a.jpg'}                 \n",
       "448  {'fullName': 'Garrett', 'avatar': 'https://cdn2.scratch.mit.edu/get_image/gallery/2090239_170x100.png'}               \n",
       "248  {'fullName': 'Leesha', 'avatar': 'https://imgbin.com/png/wKHccVQ4/pac-man-cherry-post-it-note-t-shirt-sticker-png'}   \n",
       "344  {'fullName': 'Kapil Garg', 'avatar': 'https://www.kgarg.com/img/profile.jpg'}                                         \n",
       "331  {'fullName': 'Jamie Gorson', 'avatar': 'http://tsb.northwestern.edu/img/gorson.jpg'}                                  \n",
       "\n",
       "     pairingCount  \n",
       "234  207           \n",
       "448  175           \n",
       "248  174           \n",
       "344  174           \n",
       "331  161           "
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get individuals users and their pairing sessions\n",
    "pairs_firstusers = pairs_history[['firstUserId', 'group_pairing_id']].rename(columns={'firstUserId': 'userId'})\n",
    "pairs_secondusers = pairs_history[['secondUserId', 'group_pairing_id']].rename(columns={'secondUserId': 'userId'})\n",
    "\n",
    "# aggregate\n",
    "pairs_usergroups = pd.concat([pairs_firstusers, pairs_secondusers]).dropna().drop_duplicates()\n",
    "user_pairing_count = pairs_usergroups.groupby('userId')['group_pairing_id'].size().to_frame(name = 'pairingCount').reset_index()\n",
    "\n",
    "# add in people's names\n",
    "user_pairing_count_named = user_pairing_count.merge(users, left_on='userId', right_on='_id')[['userId', 'profile', 'pairingCount']]\n",
    "user_pairing_count_named.sort_values('pairingCount', ascending=False, inplace=True)\n",
    "\n",
    "print('Number of pairing sessions by user')\n",
    "user_pairing_count_named.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    455.000000\n",
       "mean     11.817582 \n",
       "std      25.433244 \n",
       "min      1.000000  \n",
       "25%      1.000000  \n",
       "50%      3.000000  \n",
       "75%      11.000000 \n",
       "max      207.000000\n",
       "Name: pairingCount, dtype: float64"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# distribution of pairings per user\n",
    "user_pairing_count_named['pairingCount'].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Isolate DTR Pairing Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtr_group_id = 'sM3z5FkZfsABqcj3g'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create initial filters on the data for DTR only\n",
    "dtr_pairs_history = pairs_history.query(\"groupId == @dtr_group_id\").reset_index(drop=True)\n",
    "dtr_pairs_history = dtr_pairs_history[['group_pairing_id',\n",
    "                                       'firstUserId', 'firstUserName', 'secondUserId', 'secondUserName', 'pairing_session_index']]\n",
    "\n",
    "dtr_task_history = tasks_history.query(\"groupId == @dtr_group_id\").drop_duplicates(['task', 'userId', 'group_pairing_id']).reset_index(drop=True)\n",
    "\n",
    "dtr_affinities_history = affinities_history.query(\"groupId == @dtr_group_id\").reset_index(drop=True)\n",
    "dtr_affinities_history = dtr_affinities_history[['group_pairing_id', 'helpeeId', 'helperId', 'value', 'pairing_session_index']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# duplicate pairs data for non-nan entries\n",
    "dtr_pairs_history_dup = dtr_pairs_history[~dtr_pairs_history['secondUserId'].isnull()]\n",
    "dtr_pairs_history_dup = dtr_pairs_history_dup[['group_pairing_id', 'secondUserId', 'secondUserName', 'firstUserId', 'firstUserName', 'pairing_session_index']]\n",
    "dtr_pairs_history_dup.columns = ['group_pairing_id', 'helperId', 'helperName', 'helpeeId', 'helpeeName', 'pairing_session_index']\n",
    "\n",
    "# combine back with original dtr_pairs_history df\n",
    "dtr_pairs_history.columns = ['group_pairing_id', 'helperId', 'helperName', 'helpeeId', 'helpeeName', 'pairing_session_index']\n",
    "dtr_pairs_history = dtr_pairs_history.append(dtr_pairs_history_dup, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>group_pairing_id</th>\n",
       "      <th>helperId</th>\n",
       "      <th>helperName</th>\n",
       "      <th>helpeeId</th>\n",
       "      <th>helpeeName</th>\n",
       "      <th>pairing_session_index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-QrbK5SDrfk75E67qj</td>\n",
       "      <td>6rZbYn3cbQ9KNLRM5</td>\n",
       "      <td>Meg Grasse</td>\n",
       "      <td>aNdSTecskgeAm2St5</td>\n",
       "      <td>Leesha</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-QrbK5SDrfk75E67qj</td>\n",
       "      <td>EDEFWcagLwCfXP5Jg</td>\n",
       "      <td>Yongsung Kim</td>\n",
       "      <td>Rse39xrxtP6xRHsbK</td>\n",
       "      <td>AlainaKafkes2017@u.northwestern.edu</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-QrbK5SDrfk75E67qj</td>\n",
       "      <td>N3LsK5NJeKm8PkBx8</td>\n",
       "      <td>Allison Sun</td>\n",
       "      <td>mdhFQ6PNiAhfP7ce2</td>\n",
       "      <td>Kapil Garg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-QrbK5SDrfk75E67qj</td>\n",
       "      <td>Rse39xrxtP6xRHsbK</td>\n",
       "      <td>AlainaKafkes2017@u.northwestern.edu</td>\n",
       "      <td>EDEFWcagLwCfXP5Jg</td>\n",
       "      <td>Yongsung Kim</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-QrbK5SDrfk75E67qj</td>\n",
       "      <td>TFtNL3sYbSSGykQJE</td>\n",
       "      <td>Ryan Madden</td>\n",
       "      <td>aupdNzYu8WmNEi4e5</td>\n",
       "      <td>Alex Kaldjian</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-QrbK5SDrfk75E67qj</td>\n",
       "      <td>XjH8WJxEXFgTwdi3o</td>\n",
       "      <td>Sameer Srivastava</td>\n",
       "      <td>nDHZGzczDWyqvyFhp</td>\n",
       "      <td>Sarah Lim</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-QrbK5SDrfk75E67qj</td>\n",
       "      <td>aNdSTecskgeAm2St5</td>\n",
       "      <td>Leesha</td>\n",
       "      <td>6rZbYn3cbQ9KNLRM5</td>\n",
       "      <td>Meg Grasse</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-QrbK5SDrfk75E67qj</td>\n",
       "      <td>aupdNzYu8WmNEi4e5</td>\n",
       "      <td>Alex Kaldjian</td>\n",
       "      <td>TFtNL3sYbSSGykQJE</td>\n",
       "      <td>Ryan Madden</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-QrbK5SDrfk75E67qj</td>\n",
       "      <td>cupTmXWNEZ2N7vxPk</td>\n",
       "      <td>Katie George</td>\n",
       "      <td>iEHKgJBH7hNSroEjw</td>\n",
       "      <td>Greg Kim</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-QrbK5SDrfk75E67qj</td>\n",
       "      <td>iEHKgJBH7hNSroEjw</td>\n",
       "      <td>Greg Kim</td>\n",
       "      <td>cupTmXWNEZ2N7vxPk</td>\n",
       "      <td>Katie George</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      group_pairing_id           helperId  \\\n",
       "0  sM3z5FkZfsABqcj3g-QrbK5SDrfk75E67qj  6rZbYn3cbQ9KNLRM5   \n",
       "1  sM3z5FkZfsABqcj3g-QrbK5SDrfk75E67qj  EDEFWcagLwCfXP5Jg   \n",
       "2  sM3z5FkZfsABqcj3g-QrbK5SDrfk75E67qj  N3LsK5NJeKm8PkBx8   \n",
       "3  sM3z5FkZfsABqcj3g-QrbK5SDrfk75E67qj  Rse39xrxtP6xRHsbK   \n",
       "4  sM3z5FkZfsABqcj3g-QrbK5SDrfk75E67qj  TFtNL3sYbSSGykQJE   \n",
       "5  sM3z5FkZfsABqcj3g-QrbK5SDrfk75E67qj  XjH8WJxEXFgTwdi3o   \n",
       "6  sM3z5FkZfsABqcj3g-QrbK5SDrfk75E67qj  aNdSTecskgeAm2St5   \n",
       "7  sM3z5FkZfsABqcj3g-QrbK5SDrfk75E67qj  aupdNzYu8WmNEi4e5   \n",
       "8  sM3z5FkZfsABqcj3g-QrbK5SDrfk75E67qj  cupTmXWNEZ2N7vxPk   \n",
       "9  sM3z5FkZfsABqcj3g-QrbK5SDrfk75E67qj  iEHKgJBH7hNSroEjw   \n",
       "\n",
       "                            helperName           helpeeId  \\\n",
       "0  Meg Grasse                           aNdSTecskgeAm2St5   \n",
       "1  Yongsung Kim                         Rse39xrxtP6xRHsbK   \n",
       "2  Allison Sun                          mdhFQ6PNiAhfP7ce2   \n",
       "3  AlainaKafkes2017@u.northwestern.edu  EDEFWcagLwCfXP5Jg   \n",
       "4  Ryan Madden                          aupdNzYu8WmNEi4e5   \n",
       "5  Sameer Srivastava                    nDHZGzczDWyqvyFhp   \n",
       "6  Leesha                               6rZbYn3cbQ9KNLRM5   \n",
       "7  Alex Kaldjian                        TFtNL3sYbSSGykQJE   \n",
       "8  Katie George                         iEHKgJBH7hNSroEjw   \n",
       "9  Greg Kim                             cupTmXWNEZ2N7vxPk   \n",
       "\n",
       "                            helpeeName  pairing_session_index  \n",
       "0  Leesha                               1                      \n",
       "1  AlainaKafkes2017@u.northwestern.edu  1                      \n",
       "2  Kapil Garg                           1                      \n",
       "3  Yongsung Kim                         1                      \n",
       "4  Alex Kaldjian                        1                      \n",
       "5  Sarah Lim                            1                      \n",
       "6  Meg Grasse                           1                      \n",
       "7  Ryan Madden                          1                      \n",
       "8  Greg Kim                             1                      \n",
       "9  Katie George                         1                      "
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dtr_pairs_history = dtr_pairs_history.sort_values(['pairing_session_index', 'group_pairing_id', 'helperId']).reset_index(drop=True)\n",
    "dtr_pairs_history.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>group_pairing_id</th>\n",
       "      <th>helperId</th>\n",
       "      <th>helperName</th>\n",
       "      <th>helpeeId</th>\n",
       "      <th>helpeeName</th>\n",
       "      <th>pairing_session_index</th>\n",
       "      <th>task</th>\n",
       "      <th>paired</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-QrbK5SDrfk75E67qj</td>\n",
       "      <td>6rZbYn3cbQ9KNLRM5</td>\n",
       "      <td>Meg Grasse</td>\n",
       "      <td>aNdSTecskgeAm2St5</td>\n",
       "      <td>Leesha</td>\n",
       "      <td>1</td>\n",
       "      <td>I need to think through &amp; implement test cases for automated runner tracking.</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-QrbK5SDrfk75E67qj</td>\n",
       "      <td>EDEFWcagLwCfXP5Jg</td>\n",
       "      <td>Yongsung Kim</td>\n",
       "      <td>Rse39xrxtP6xRHsbK</td>\n",
       "      <td>AlainaKafkes2017@u.northwestern.edu</td>\n",
       "      <td>1</td>\n",
       "      <td>Scaffolding feedback</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-QrbK5SDrfk75E67qj</td>\n",
       "      <td>N3LsK5NJeKm8PkBx8</td>\n",
       "      <td>Allison Sun</td>\n",
       "      <td>mdhFQ6PNiAhfP7ce2</td>\n",
       "      <td>Kapil Garg</td>\n",
       "      <td>1</td>\n",
       "      <td>Finish up scenarios and study design</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-QrbK5SDrfk75E67qj</td>\n",
       "      <td>Rse39xrxtP6xRHsbK</td>\n",
       "      <td>AlainaKafkes2017@u.northwestern.edu</td>\n",
       "      <td>EDEFWcagLwCfXP5Jg</td>\n",
       "      <td>Yongsung Kim</td>\n",
       "      <td>1</td>\n",
       "      <td>i need help with my fellowship research statement</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-QrbK5SDrfk75E67qj</td>\n",
       "      <td>TFtNL3sYbSSGykQJE</td>\n",
       "      <td>Ryan Madden</td>\n",
       "      <td>aupdNzYu8WmNEi4e5</td>\n",
       "      <td>Alex Kaldjian</td>\n",
       "      <td>1</td>\n",
       "      <td>coming up with prototyping examples for exploratory study</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      group_pairing_id           helperId  \\\n",
       "0  sM3z5FkZfsABqcj3g-QrbK5SDrfk75E67qj  6rZbYn3cbQ9KNLRM5   \n",
       "1  sM3z5FkZfsABqcj3g-QrbK5SDrfk75E67qj  EDEFWcagLwCfXP5Jg   \n",
       "2  sM3z5FkZfsABqcj3g-QrbK5SDrfk75E67qj  N3LsK5NJeKm8PkBx8   \n",
       "3  sM3z5FkZfsABqcj3g-QrbK5SDrfk75E67qj  Rse39xrxtP6xRHsbK   \n",
       "4  sM3z5FkZfsABqcj3g-QrbK5SDrfk75E67qj  TFtNL3sYbSSGykQJE   \n",
       "\n",
       "                            helperName           helpeeId  \\\n",
       "0  Meg Grasse                           aNdSTecskgeAm2St5   \n",
       "1  Yongsung Kim                         Rse39xrxtP6xRHsbK   \n",
       "2  Allison Sun                          mdhFQ6PNiAhfP7ce2   \n",
       "3  AlainaKafkes2017@u.northwestern.edu  EDEFWcagLwCfXP5Jg   \n",
       "4  Ryan Madden                          aupdNzYu8WmNEi4e5   \n",
       "\n",
       "                            helpeeName  pairing_session_index  \\\n",
       "0  Leesha                               1                       \n",
       "1  AlainaKafkes2017@u.northwestern.edu  1                       \n",
       "2  Kapil Garg                           1                       \n",
       "3  Yongsung Kim                         1                       \n",
       "4  Alex Kaldjian                        1                       \n",
       "\n",
       "                                                                            task  \\\n",
       "0  I need to think through & implement test cases for automated runner tracking.   \n",
       "1  Scaffolding feedback                                                            \n",
       "2  Finish up scenarios and study design                                            \n",
       "3  i need help with my fellowship research statement                               \n",
       "4  coming up with prototyping examples for exploratory study                       \n",
       "\n",
       "   paired  \n",
       "0  True    \n",
       "1  True    \n",
       "2  True    \n",
       "3  True    \n",
       "4  True    "
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# add task requests made by the helpee\n",
    "dtr_pairs_tasks = dtr_pairs_history.merge(dtr_task_history[['group_pairing_id', 'userId', 'task', 'pairing_session_index']],\n",
    "                                          left_on=['group_pairing_id', 'helpeeId', 'pairing_session_index'],\n",
    "                                          right_on=['group_pairing_id', 'userId', 'pairing_session_index'],\n",
    "                                          how='left')\n",
    "dtr_pairs_tasks['paired'] = True\n",
    "del dtr_pairs_tasks['userId']\n",
    "dtr_pairs_tasks.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>group_pairing_id</th>\n",
       "      <th>helperId</th>\n",
       "      <th>helperName</th>\n",
       "      <th>helpeeId</th>\n",
       "      <th>helpeeName</th>\n",
       "      <th>pairing_session_index</th>\n",
       "      <th>paired</th>\n",
       "      <th>value</th>\n",
       "      <th>userId</th>\n",
       "      <th>task</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-2CgSkRTMERna4KKdX</td>\n",
       "      <td>5EJncSsf5AStw6maE</td>\n",
       "      <td>Nneoma Oradiegwu</td>\n",
       "      <td>3cb7eEv3gY3xiBCk7</td>\n",
       "      <td>Daniel Zhu</td>\n",
       "      <td>24</td>\n",
       "      <td>False</td>\n",
       "      <td>0.00</td>\n",
       "      <td>3cb7eEv3gY3xiBCk7</td>\n",
       "      <td>Could someone go through my research canvas with me and give me a critique of my design arguments + gaps I should fill in?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-2CgSkRTMERna4KKdX</td>\n",
       "      <td>9Wcpa2nSnEX8k7kMN</td>\n",
       "      <td>Maggie Lou</td>\n",
       "      <td>3cb7eEv3gY3xiBCk7</td>\n",
       "      <td>Daniel Zhu</td>\n",
       "      <td>24</td>\n",
       "      <td>False</td>\n",
       "      <td>0.66</td>\n",
       "      <td>3cb7eEv3gY3xiBCk7</td>\n",
       "      <td>Could someone go through my research canvas with me and give me a critique of my design arguments + gaps I should fill in?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-2CgSkRTMERna4KKdX</td>\n",
       "      <td>9gcjpT4daYZmZDi2F</td>\n",
       "      <td>Armaan Shah</td>\n",
       "      <td>3cb7eEv3gY3xiBCk7</td>\n",
       "      <td>Daniel Zhu</td>\n",
       "      <td>24</td>\n",
       "      <td>False</td>\n",
       "      <td>0.33</td>\n",
       "      <td>3cb7eEv3gY3xiBCk7</td>\n",
       "      <td>Could someone go through my research canvas with me and give me a critique of my design arguments + gaps I should fill in?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-2CgSkRTMERna4KKdX</td>\n",
       "      <td>HnvDuWqAnAqHbXxLT</td>\n",
       "      <td>Sanfeng Wang</td>\n",
       "      <td>3cb7eEv3gY3xiBCk7</td>\n",
       "      <td>Daniel Zhu</td>\n",
       "      <td>24</td>\n",
       "      <td>True</td>\n",
       "      <td>0.66</td>\n",
       "      <td>3cb7eEv3gY3xiBCk7</td>\n",
       "      <td>Could someone go through my research canvas with me and give me a critique of my design arguments + gaps I should fill in?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-2CgSkRTMERna4KKdX</td>\n",
       "      <td>LG96v2nkrkJhWstC4</td>\n",
       "      <td>Olivia Barnett</td>\n",
       "      <td>3cb7eEv3gY3xiBCk7</td>\n",
       "      <td>Daniel Zhu</td>\n",
       "      <td>24</td>\n",
       "      <td>False</td>\n",
       "      <td>0.00</td>\n",
       "      <td>3cb7eEv3gY3xiBCk7</td>\n",
       "      <td>Could someone go through my research canvas with me and give me a critique of my design arguments + gaps I should fill in?</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      group_pairing_id           helperId        helperName  \\\n",
       "0  sM3z5FkZfsABqcj3g-2CgSkRTMERna4KKdX  5EJncSsf5AStw6maE  Nneoma Oradiegwu   \n",
       "1  sM3z5FkZfsABqcj3g-2CgSkRTMERna4KKdX  9Wcpa2nSnEX8k7kMN  Maggie Lou         \n",
       "2  sM3z5FkZfsABqcj3g-2CgSkRTMERna4KKdX  9gcjpT4daYZmZDi2F  Armaan Shah        \n",
       "3  sM3z5FkZfsABqcj3g-2CgSkRTMERna4KKdX  HnvDuWqAnAqHbXxLT  Sanfeng Wang       \n",
       "4  sM3z5FkZfsABqcj3g-2CgSkRTMERna4KKdX  LG96v2nkrkJhWstC4  Olivia Barnett     \n",
       "\n",
       "            helpeeId  helpeeName  pairing_session_index paired  value  \\\n",
       "0  3cb7eEv3gY3xiBCk7  Daniel Zhu  24                     False  0.00    \n",
       "1  3cb7eEv3gY3xiBCk7  Daniel Zhu  24                     False  0.66    \n",
       "2  3cb7eEv3gY3xiBCk7  Daniel Zhu  24                     False  0.33    \n",
       "3  3cb7eEv3gY3xiBCk7  Daniel Zhu  24                     True   0.66    \n",
       "4  3cb7eEv3gY3xiBCk7  Daniel Zhu  24                     False  0.00    \n",
       "\n",
       "              userId  \\\n",
       "0  3cb7eEv3gY3xiBCk7   \n",
       "1  3cb7eEv3gY3xiBCk7   \n",
       "2  3cb7eEv3gY3xiBCk7   \n",
       "3  3cb7eEv3gY3xiBCk7   \n",
       "4  3cb7eEv3gY3xiBCk7   \n",
       "\n",
       "                                                                                                                         task  \n",
       "0  Could someone go through my research canvas with me and give me a critique of my design arguments + gaps I should fill in?  \n",
       "1  Could someone go through my research canvas with me and give me a critique of my design arguments + gaps I should fill in?  \n",
       "2  Could someone go through my research canvas with me and give me a critique of my design arguments + gaps I should fill in?  \n",
       "3  Could someone go through my research canvas with me and give me a critique of my design arguments + gaps I should fill in?  \n",
       "4  Could someone go through my research canvas with me and give me a critique of my design arguments + gaps I should fill in?  "
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# add helper's ability to help. keep all affinities even if people weren't actually paired (how=right; for only paired people, use how='left')\n",
    "dtr_pairs_tasks_affinities = dtr_pairs_tasks.merge(dtr_affinities_history,\n",
    "                                                   left_on=['group_pairing_id', 'helperId', 'helpeeId', 'pairing_session_index'],\n",
    "                                                   right_on=['group_pairing_id', 'helperId', 'helpeeId', 'pairing_session_index'],\n",
    "                                                   how='right')\n",
    "\n",
    "# fill with 0 affinity if person has no partner\n",
    "dtr_pairs_tasks_affinities.loc[(~dtr_pairs_tasks_affinities['helpeeId'].isnull()) & (dtr_pairs_tasks_affinities['value'].isnull()), 'value'] = 0\n",
    "\n",
    "# set paired to false for any unparied people (i.e., all rated affinties for people who weren't paired)\n",
    "dtr_pairs_tasks_affinities.loc[dtr_pairs_tasks_affinities.paired.isnull(), 'paired'] = False\n",
    "\n",
    "# fill blank names\n",
    "dtr_name_dict = {row['userId']: row['name'] for index, row in dtr_task_history[['userId',  'name']].iterrows()}\n",
    "dtr_pairs_tasks_affinities['helperName'] = dtr_pairs_tasks_affinities['helperName'].fillna(dtr_pairs_tasks_affinities['helperId']).replace(dtr_name_dict)\n",
    "dtr_pairs_tasks_affinities['helpeeName'] = dtr_pairs_tasks_affinities['helpeeName'].fillna(dtr_pairs_tasks_affinities['helpeeId']).replace(dtr_name_dict)\n",
    "\n",
    "# fill in blank tasks\n",
    "dtr_pairs_tasks_affinities = dtr_pairs_tasks_affinities.merge(dtr_task_history[['group_pairing_id', 'userId', 'task']],\n",
    "                                                              left_on=['group_pairing_id', 'helpeeId'],\n",
    "                                                              right_on=['group_pairing_id', 'userId'])\n",
    "del dtr_pairs_tasks_affinities['task_x']\n",
    "dtr_pairs_tasks_affinities.rename(columns={'task_y': 'task'}, inplace=True)\n",
    "\n",
    "# sort table and print\n",
    "dtr_pairs_tasks_affinities.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# replace values with 1 - 5\n",
    "value_mappings = {\n",
    "    '-1.0': 1,\n",
    "    '0.0':  2,\n",
    "    '0.33': 3,\n",
    "    '0.66': 4,\n",
    "    '1.0':  5\n",
    "}\n",
    "\n",
    "dtr_pairs_tasks_affinities['value'] = dtr_pairs_tasks_affinities['value'].astype(str)\n",
    "dtr_pairs_tasks_affinities.replace({'value': value_mappings}, inplace=True)\n",
    "\n",
    "# replace names\n",
    "name_mappings = {\n",
    "    'richardhuang2019@u.northwestern.edu': 'Richard Huang',\n",
    "    'AlainaKafkes2017@u.northwestern.edu': 'Alaina Kafkes',\n",
    "    'judylee2021@u.northwestern.edu': 'Judy Lee',\n",
    "    'Leesha': 'Leesha Maliakal',\n",
    "    'andrew': 'Andrew Finke',\n",
    "    'Garrett': 'Garrett Hedman'\n",
    "}\n",
    "dtr_pairs_tasks_affinities.replace({'helperName': name_mappings, 'helpeeName': name_mappings}, inplace=True)\n",
    "\n",
    "# more intutive column names\n",
    "dtr_pairs_tasks_affinities.rename(columns={'task': 'helpeeRequest', 'value': 'helperAbilityToHelp'}, inplace=True)\n",
    "\n",
    "# reorder columns\n",
    "dtr_pairs_tasks_affinities = dtr_pairs_tasks_affinities[['group_pairing_id', 'pairing_session_index', 'helperId', 'helperName',\n",
    "                                                         'helpeeId', 'helpeeName', 'helpeeRequest', 'helperAbilityToHelp', 'paired']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove nans for initial analysis, and save out csv\n",
    "dtr_pairs_tasks_affinities.dropna(how='any',axis=0).to_csv('./documents/dtr-pair-research-pairings.csv',\n",
    "                                                           index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>group_pairing_id</th>\n",
       "      <th>pairing_session_index</th>\n",
       "      <th>helperId</th>\n",
       "      <th>helperName</th>\n",
       "      <th>helpeeId</th>\n",
       "      <th>helpeeName</th>\n",
       "      <th>helpeeRequest</th>\n",
       "      <th>helperAbilityToHelp</th>\n",
       "      <th>paired</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-2CgSkRTMERna4KKdX</td>\n",
       "      <td>24</td>\n",
       "      <td>5EJncSsf5AStw6maE</td>\n",
       "      <td>Nneoma Oradiegwu</td>\n",
       "      <td>3cb7eEv3gY3xiBCk7</td>\n",
       "      <td>Daniel Zhu</td>\n",
       "      <td>Could someone go through my research canvas with me and give me a critique of my design arguments + gaps I should fill in?</td>\n",
       "      <td>2</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-2CgSkRTMERna4KKdX</td>\n",
       "      <td>24</td>\n",
       "      <td>9Wcpa2nSnEX8k7kMN</td>\n",
       "      <td>Maggie Lou</td>\n",
       "      <td>3cb7eEv3gY3xiBCk7</td>\n",
       "      <td>Daniel Zhu</td>\n",
       "      <td>Could someone go through my research canvas with me and give me a critique of my design arguments + gaps I should fill in?</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-2CgSkRTMERna4KKdX</td>\n",
       "      <td>24</td>\n",
       "      <td>9gcjpT4daYZmZDi2F</td>\n",
       "      <td>Armaan Shah</td>\n",
       "      <td>3cb7eEv3gY3xiBCk7</td>\n",
       "      <td>Daniel Zhu</td>\n",
       "      <td>Could someone go through my research canvas with me and give me a critique of my design arguments + gaps I should fill in?</td>\n",
       "      <td>3</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-2CgSkRTMERna4KKdX</td>\n",
       "      <td>24</td>\n",
       "      <td>HnvDuWqAnAqHbXxLT</td>\n",
       "      <td>Sanfeng Wang</td>\n",
       "      <td>3cb7eEv3gY3xiBCk7</td>\n",
       "      <td>Daniel Zhu</td>\n",
       "      <td>Could someone go through my research canvas with me and give me a critique of my design arguments + gaps I should fill in?</td>\n",
       "      <td>4</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-2CgSkRTMERna4KKdX</td>\n",
       "      <td>24</td>\n",
       "      <td>LG96v2nkrkJhWstC4</td>\n",
       "      <td>Olivia Barnett</td>\n",
       "      <td>3cb7eEv3gY3xiBCk7</td>\n",
       "      <td>Daniel Zhu</td>\n",
       "      <td>Could someone go through my research canvas with me and give me a critique of my design arguments + gaps I should fill in?</td>\n",
       "      <td>2</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21623</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-zxRFfvAsSTD9MQGWa</td>\n",
       "      <td>91</td>\n",
       "      <td>fSsiAiF3qdkhdgYqk</td>\n",
       "      <td>Kieran Bondy</td>\n",
       "      <td>zmwK4tJHtwLw8pLRC</td>\n",
       "      <td>Garrett Hedman</td>\n",
       "      <td>Ask me to explain my research to you (so I can figure out how to best structure an explanation of my research) :D</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21624</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-zxRFfvAsSTD9MQGWa</td>\n",
       "      <td>91</td>\n",
       "      <td>ia6Mh9bm7Wg3K4pkc</td>\n",
       "      <td>Mary Truong</td>\n",
       "      <td>zmwK4tJHtwLw8pLRC</td>\n",
       "      <td>Garrett Hedman</td>\n",
       "      <td>Ask me to explain my research to you (so I can figure out how to best structure an explanation of my research) :D</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21625</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-zxRFfvAsSTD9MQGWa</td>\n",
       "      <td>91</td>\n",
       "      <td>jxctk3DqhWorsPMXS</td>\n",
       "      <td>Natalie Ghidali</td>\n",
       "      <td>zmwK4tJHtwLw8pLRC</td>\n",
       "      <td>Garrett Hedman</td>\n",
       "      <td>Ask me to explain my research to you (so I can figure out how to best structure an explanation of my research) :D</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21626</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-zxRFfvAsSTD9MQGWa</td>\n",
       "      <td>91</td>\n",
       "      <td>mdhFQ6PNiAhfP7ce2</td>\n",
       "      <td>Kapil Garg</td>\n",
       "      <td>zmwK4tJHtwLw8pLRC</td>\n",
       "      <td>Garrett Hedman</td>\n",
       "      <td>Ask me to explain my research to you (so I can figure out how to best structure an explanation of my research) :D</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21627</th>\n",
       "      <td>sM3z5FkZfsABqcj3g-zxRFfvAsSTD9MQGWa</td>\n",
       "      <td>91</td>\n",
       "      <td>p4E6kyobhskytDdLX</td>\n",
       "      <td>Cooper Barth</td>\n",
       "      <td>zmwK4tJHtwLw8pLRC</td>\n",
       "      <td>Garrett Hedman</td>\n",
       "      <td>Ask me to explain my research to you (so I can figure out how to best structure an explanation of my research) :D</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>21628 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                          group_pairing_id  pairing_session_index  \\\n",
       "0      sM3z5FkZfsABqcj3g-2CgSkRTMERna4KKdX  24                      \n",
       "1      sM3z5FkZfsABqcj3g-2CgSkRTMERna4KKdX  24                      \n",
       "2      sM3z5FkZfsABqcj3g-2CgSkRTMERna4KKdX  24                      \n",
       "3      sM3z5FkZfsABqcj3g-2CgSkRTMERna4KKdX  24                      \n",
       "4      sM3z5FkZfsABqcj3g-2CgSkRTMERna4KKdX  24                      \n",
       "...                                    ...  ..                      \n",
       "21623  sM3z5FkZfsABqcj3g-zxRFfvAsSTD9MQGWa  91                      \n",
       "21624  sM3z5FkZfsABqcj3g-zxRFfvAsSTD9MQGWa  91                      \n",
       "21625  sM3z5FkZfsABqcj3g-zxRFfvAsSTD9MQGWa  91                      \n",
       "21626  sM3z5FkZfsABqcj3g-zxRFfvAsSTD9MQGWa  91                      \n",
       "21627  sM3z5FkZfsABqcj3g-zxRFfvAsSTD9MQGWa  91                      \n",
       "\n",
       "                helperId        helperName           helpeeId      helpeeName  \\\n",
       "0      5EJncSsf5AStw6maE  Nneoma Oradiegwu  3cb7eEv3gY3xiBCk7  Daniel Zhu       \n",
       "1      9Wcpa2nSnEX8k7kMN  Maggie Lou        3cb7eEv3gY3xiBCk7  Daniel Zhu       \n",
       "2      9gcjpT4daYZmZDi2F  Armaan Shah       3cb7eEv3gY3xiBCk7  Daniel Zhu       \n",
       "3      HnvDuWqAnAqHbXxLT  Sanfeng Wang      3cb7eEv3gY3xiBCk7  Daniel Zhu       \n",
       "4      LG96v2nkrkJhWstC4  Olivia Barnett    3cb7eEv3gY3xiBCk7  Daniel Zhu       \n",
       "...                  ...             ...                  ...         ...       \n",
       "21623  fSsiAiF3qdkhdgYqk  Kieran Bondy      zmwK4tJHtwLw8pLRC  Garrett Hedman   \n",
       "21624  ia6Mh9bm7Wg3K4pkc  Mary Truong       zmwK4tJHtwLw8pLRC  Garrett Hedman   \n",
       "21625  jxctk3DqhWorsPMXS  Natalie Ghidali   zmwK4tJHtwLw8pLRC  Garrett Hedman   \n",
       "21626  mdhFQ6PNiAhfP7ce2  Kapil Garg        zmwK4tJHtwLw8pLRC  Garrett Hedman   \n",
       "21627  p4E6kyobhskytDdLX  Cooper Barth      zmwK4tJHtwLw8pLRC  Garrett Hedman   \n",
       "\n",
       "                                                                                                                    helpeeRequest  \\\n",
       "0      Could someone go through my research canvas with me and give me a critique of my design arguments + gaps I should fill in?   \n",
       "1      Could someone go through my research canvas with me and give me a critique of my design arguments + gaps I should fill in?   \n",
       "2      Could someone go through my research canvas with me and give me a critique of my design arguments + gaps I should fill in?   \n",
       "3      Could someone go through my research canvas with me and give me a critique of my design arguments + gaps I should fill in?   \n",
       "4      Could someone go through my research canvas with me and give me a critique of my design arguments + gaps I should fill in?   \n",
       "...                                                                                                                           ...   \n",
       "21623  Ask me to explain my research to you (so I can figure out how to best structure an explanation of my research) :D            \n",
       "21624  Ask me to explain my research to you (so I can figure out how to best structure an explanation of my research) :D            \n",
       "21625  Ask me to explain my research to you (so I can figure out how to best structure an explanation of my research) :D            \n",
       "21626  Ask me to explain my research to you (so I can figure out how to best structure an explanation of my research) :D            \n",
       "21627  Ask me to explain my research to you (so I can figure out how to best structure an explanation of my research) :D            \n",
       "\n",
       "       helperAbilityToHelp paired  \n",
       "0      2                    False  \n",
       "1      4                    False  \n",
       "2      3                    False  \n",
       "3      4                    True   \n",
       "4      2                    False  \n",
       "...   ..                      ...  \n",
       "21623  1                    False  \n",
       "21624  5                    False  \n",
       "21625  5                    False  \n",
       "21626  4                    False  \n",
       "21627  4                    False  \n",
       "\n",
       "[21628 rows x 9 columns]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dtr_pairs_tasks_affinities"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summary Stats from DTR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Pairing Sessions: 126\n"
     ]
    }
   ],
   "source": [
    "# number of pairing sessions\n",
    "print('Number of Pairing Sessions: {}'.format(len(dtr_pairs_history['group_pairing_id'].unique())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Unique Users: 92\n"
     ]
    }
   ],
   "source": [
    "# number of unique users\n",
    "print('Number of Unique Users: {}'.format(len(set(dtr_pairs_history['helperId'].unique()).union(set(dtr_pairs_history['helpeeId'].unique())))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of tasks: 1765\n"
     ]
    }
   ],
   "source": [
    "# total number of tasks\n",
    "print('Total number of tasks: {}'.format(len(dtr_task_history)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    126.000000\n",
       "mean     14.007937 \n",
       "std      4.091936  \n",
       "min      1.000000  \n",
       "25%      12.000000 \n",
       "50%      14.000000 \n",
       "75%      17.000000 \n",
       "max      24.000000 \n",
       "Name: userId, dtype: float64"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# number of users/tasks per session\n",
    "dtr_task_history.groupby('group_pairing_id')['userId'].count().describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of affinities: 21628\n"
     ]
    }
   ],
   "source": [
    "# total number of specified affinities\n",
    "print('Total number of affinities: {}'.format(len(dtr_affinities_history)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    125.000000\n",
       "mean     173.024000\n",
       "std      97.348183 \n",
       "min      12.000000 \n",
       "25%      108.000000\n",
       "50%      150.000000\n",
       "75%      227.000000\n",
       "max      535.000000\n",
       "Name: value, dtype: float64"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# number of affinities per session\n",
    "dtr_affinities_history.groupby(['group_pairing_id'])['value'].count().describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    1696.000000\n",
       "mean     12.752358  \n",
       "std      4.379221   \n",
       "min      1.000000   \n",
       "25%      10.000000  \n",
       "50%      12.000000  \n",
       "75%      16.000000  \n",
       "max      23.000000  \n",
       "Name: value, dtype: float64"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# number of affinities per user per session\n",
    "dtr_affinities_history.groupby(['group_pairing_id', 'helperId'])['value'].count().describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Isolate Delta Lab Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "delta_group_id = '9mdkMmj4pY8Q2TwqF'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create initial filters on the data for Delta only\n",
    "delta_pairs_history = pairs_history.query(\"groupId == @delta_group_id\").reset_index(drop=True)\n",
    "delta_pairs_history = delta_pairs_history[['group_pairing_id',\n",
    "                                       'firstUserId', 'firstUserName', 'secondUserId', 'secondUserName', 'pairing_session_index']]\n",
    "\n",
    "delta_task_history = tasks_history.query(\"groupId == @delta_group_id\").drop_duplicates(['task', 'userId', 'group_pairing_id']).reset_index(drop=True)\n",
    "\n",
    "delta_affinities_history = affinities_history.query(\"groupId == @delta_group_id\").reset_index(drop=True)\n",
    "delta_affinities_history = delta_affinities_history[['group_pairing_id', 'helpeeId', 'helperId', 'value', 'pairing_session_index']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# duplicate pairs data for non-nan entries\n",
    "delta_pairs_history_dup = delta_pairs_history[~delta_pairs_history['secondUserId'].isnull()]\n",
    "delta_pairs_history_dup = delta_pairs_history_dup[['group_pairing_id', 'secondUserId', 'secondUserName', 'firstUserId', 'firstUserName', 'pairing_session_index']]\n",
    "delta_pairs_history_dup.columns = ['group_pairing_id', 'helperId', 'helperName', 'helpeeId', 'helpeeName', 'pairing_session_index']\n",
    "\n",
    "# combine back with original delta_pairs_history df\n",
    "delta_pairs_history.columns = ['group_pairing_id', 'helperId', 'helperName', 'helpeeId', 'helpeeName', 'pairing_session_index']\n",
    "delta_pairs_history = delta_pairs_history.append(delta_pairs_history_dup, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>group_pairing_id</th>\n",
       "      <th>helperId</th>\n",
       "      <th>helperName</th>\n",
       "      <th>helpeeId</th>\n",
       "      <th>helpeeName</th>\n",
       "      <th>pairing_session_index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-N23iLvjp2GWcsHYd5</td>\n",
       "      <td>57MnWENtTDkXRYhcL</td>\n",
       "      <td>Gulu</td>\n",
       "      <td>zBZSGgrZFfW5KH5vj</td>\n",
       "      <td>Natalia Smirnov</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-N23iLvjp2GWcsHYd5</td>\n",
       "      <td>6iR9Z64HEJDcD8qbu</td>\n",
       "      <td>Matt Easterday</td>\n",
       "      <td>MJkj24zXWKhnZQCc3</td>\n",
       "      <td>Daniel George Rees Lewis</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-N23iLvjp2GWcsHYd5</td>\n",
       "      <td>EDEFWcagLwCfXP5Jg</td>\n",
       "      <td>Yongsung Kim</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-N23iLvjp2GWcsHYd5</td>\n",
       "      <td>JaEySKdKKg7LAF3Yg</td>\n",
       "      <td>Eureka Foong</td>\n",
       "      <td>gynuaAvfp3gAd4Gyo</td>\n",
       "      <td>eharburg@gmail.com</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-N23iLvjp2GWcsHYd5</td>\n",
       "      <td>KYnkykoMwd9fbBbWB</td>\n",
       "      <td>Julie Hui</td>\n",
       "      <td>aNdSTecskgeAm2St5</td>\n",
       "      <td>Leesha</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-N23iLvjp2GWcsHYd5</td>\n",
       "      <td>MJkj24zXWKhnZQCc3</td>\n",
       "      <td>Daniel George Rees Lewis</td>\n",
       "      <td>6iR9Z64HEJDcD8qbu</td>\n",
       "      <td>Matt Easterday</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-N23iLvjp2GWcsHYd5</td>\n",
       "      <td>aNdSTecskgeAm2St5</td>\n",
       "      <td>Leesha</td>\n",
       "      <td>KYnkykoMwd9fbBbWB</td>\n",
       "      <td>Julie Hui</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-N23iLvjp2GWcsHYd5</td>\n",
       "      <td>gynuaAvfp3gAd4Gyo</td>\n",
       "      <td>eharburg@gmail.com</td>\n",
       "      <td>JaEySKdKKg7LAF3Yg</td>\n",
       "      <td>Eureka Foong</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-N23iLvjp2GWcsHYd5</td>\n",
       "      <td>zBZSGgrZFfW5KH5vj</td>\n",
       "      <td>Natalia Smirnov</td>\n",
       "      <td>57MnWENtTDkXRYhcL</td>\n",
       "      <td>Gulu</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-soiecrpv6CRPTqmkd</td>\n",
       "      <td>PavTL8zD9664wvtfB</td>\n",
       "      <td>Haoqi Zhang</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      group_pairing_id           helperId  \\\n",
       "0  9mdkMmj4pY8Q2TwqF-N23iLvjp2GWcsHYd5  57MnWENtTDkXRYhcL   \n",
       "1  9mdkMmj4pY8Q2TwqF-N23iLvjp2GWcsHYd5  6iR9Z64HEJDcD8qbu   \n",
       "2  9mdkMmj4pY8Q2TwqF-N23iLvjp2GWcsHYd5  EDEFWcagLwCfXP5Jg   \n",
       "3  9mdkMmj4pY8Q2TwqF-N23iLvjp2GWcsHYd5  JaEySKdKKg7LAF3Yg   \n",
       "4  9mdkMmj4pY8Q2TwqF-N23iLvjp2GWcsHYd5  KYnkykoMwd9fbBbWB   \n",
       "5  9mdkMmj4pY8Q2TwqF-N23iLvjp2GWcsHYd5  MJkj24zXWKhnZQCc3   \n",
       "6  9mdkMmj4pY8Q2TwqF-N23iLvjp2GWcsHYd5  aNdSTecskgeAm2St5   \n",
       "7  9mdkMmj4pY8Q2TwqF-N23iLvjp2GWcsHYd5  gynuaAvfp3gAd4Gyo   \n",
       "8  9mdkMmj4pY8Q2TwqF-N23iLvjp2GWcsHYd5  zBZSGgrZFfW5KH5vj   \n",
       "9  9mdkMmj4pY8Q2TwqF-soiecrpv6CRPTqmkd  PavTL8zD9664wvtfB   \n",
       "\n",
       "                 helperName           helpeeId                helpeeName  \\\n",
       "0  Gulu                      zBZSGgrZFfW5KH5vj  Natalia Smirnov            \n",
       "1  Matt Easterday            MJkj24zXWKhnZQCc3  Daniel George Rees Lewis   \n",
       "2  Yongsung Kim              NaN                NaN                        \n",
       "3  Eureka Foong              gynuaAvfp3gAd4Gyo  eharburg@gmail.com         \n",
       "4  Julie Hui                 aNdSTecskgeAm2St5  Leesha                     \n",
       "5  Daniel George Rees Lewis  6iR9Z64HEJDcD8qbu  Matt Easterday             \n",
       "6  Leesha                    KYnkykoMwd9fbBbWB  Julie Hui                  \n",
       "7  eharburg@gmail.com        JaEySKdKKg7LAF3Yg  Eureka Foong               \n",
       "8  Natalia Smirnov           57MnWENtTDkXRYhcL  Gulu                       \n",
       "9  Haoqi Zhang               NaN                NaN                        \n",
       "\n",
       "   pairing_session_index  \n",
       "0  1                      \n",
       "1  1                      \n",
       "2  1                      \n",
       "3  1                      \n",
       "4  1                      \n",
       "5  1                      \n",
       "6  1                      \n",
       "7  1                      \n",
       "8  1                      \n",
       "9  2                      "
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "delta_pairs_history = delta_pairs_history.sort_values(['pairing_session_index', 'group_pairing_id', 'helperId']).reset_index(drop=True)\n",
    "delta_pairs_history.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>group_pairing_id</th>\n",
       "      <th>helperId</th>\n",
       "      <th>helperName</th>\n",
       "      <th>helpeeId</th>\n",
       "      <th>helpeeName</th>\n",
       "      <th>pairing_session_index</th>\n",
       "      <th>task</th>\n",
       "      <th>paired</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-N23iLvjp2GWcsHYd5</td>\n",
       "      <td>57MnWENtTDkXRYhcL</td>\n",
       "      <td>Gulu</td>\n",
       "      <td>zBZSGgrZFfW5KH5vj</td>\n",
       "      <td>Natalia Smirnov</td>\n",
       "      <td>1</td>\n",
       "      <td>Sit next to me while I fix revisions on my paper to resubmit to a journal :]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-N23iLvjp2GWcsHYd5</td>\n",
       "      <td>6iR9Z64HEJDcD8qbu</td>\n",
       "      <td>Matt Easterday</td>\n",
       "      <td>MJkj24zXWKhnZQCc3</td>\n",
       "      <td>Daniel George Rees Lewis</td>\n",
       "      <td>1</td>\n",
       "      <td>Read CSCW R&amp;R and edit it -- also find citations that back up our proxy baseline for the study</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-N23iLvjp2GWcsHYd5</td>\n",
       "      <td>EDEFWcagLwCfXP5Jg</td>\n",
       "      <td>Yongsung Kim</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-N23iLvjp2GWcsHYd5</td>\n",
       "      <td>JaEySKdKKg7LAF3Yg</td>\n",
       "      <td>Eureka Foong</td>\n",
       "      <td>gynuaAvfp3gAd4Gyo</td>\n",
       "      <td>eharburg@gmail.com</td>\n",
       "      <td>1</td>\n",
       "      <td>Help me revise my lit for CheerOn</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-N23iLvjp2GWcsHYd5</td>\n",
       "      <td>KYnkykoMwd9fbBbWB</td>\n",
       "      <td>Julie Hui</td>\n",
       "      <td>aNdSTecskgeAm2St5</td>\n",
       "      <td>Leesha</td>\n",
       "      <td>1</td>\n",
       "      <td>I need help testing CrowdCheer outside! You just have to stand there and pretend to be a spectator, then give me feedback on the interaction.</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      group_pairing_id           helperId      helperName  \\\n",
       "0  9mdkMmj4pY8Q2TwqF-N23iLvjp2GWcsHYd5  57MnWENtTDkXRYhcL  Gulu             \n",
       "1  9mdkMmj4pY8Q2TwqF-N23iLvjp2GWcsHYd5  6iR9Z64HEJDcD8qbu  Matt Easterday   \n",
       "2  9mdkMmj4pY8Q2TwqF-N23iLvjp2GWcsHYd5  EDEFWcagLwCfXP5Jg  Yongsung Kim     \n",
       "3  9mdkMmj4pY8Q2TwqF-N23iLvjp2GWcsHYd5  JaEySKdKKg7LAF3Yg  Eureka Foong     \n",
       "4  9mdkMmj4pY8Q2TwqF-N23iLvjp2GWcsHYd5  KYnkykoMwd9fbBbWB  Julie Hui        \n",
       "\n",
       "            helpeeId                helpeeName  pairing_session_index  \\\n",
       "0  zBZSGgrZFfW5KH5vj  Natalia Smirnov           1                       \n",
       "1  MJkj24zXWKhnZQCc3  Daniel George Rees Lewis  1                       \n",
       "2  NaN                NaN                       1                       \n",
       "3  gynuaAvfp3gAd4Gyo  eharburg@gmail.com        1                       \n",
       "4  aNdSTecskgeAm2St5  Leesha                    1                       \n",
       "\n",
       "                                                                                                                                            task  \\\n",
       "0  Sit next to me while I fix revisions on my paper to resubmit to a journal :]                                                                    \n",
       "1  Read CSCW R&R and edit it -- also find citations that back up our proxy baseline for the study                                                  \n",
       "2  NaN                                                                                                                                             \n",
       "3  Help me revise my lit for CheerOn                                                                                                               \n",
       "4  I need help testing CrowdCheer outside! You just have to stand there and pretend to be a spectator, then give me feedback on the interaction.   \n",
       "\n",
       "   paired  \n",
       "0  True    \n",
       "1  True    \n",
       "2  True    \n",
       "3  True    \n",
       "4  True    "
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# add task requests made by the helpee\n",
    "delta_pairs_tasks = delta_pairs_history.merge(delta_task_history[['group_pairing_id', 'userId', 'task', 'pairing_session_index']],\n",
    "                                          left_on=['group_pairing_id', 'helpeeId', 'pairing_session_index'],\n",
    "                                          right_on=['group_pairing_id', 'userId', 'pairing_session_index'],\n",
    "                                          how='left')\n",
    "delta_pairs_tasks['paired'] = True\n",
    "del delta_pairs_tasks['userId']\n",
    "delta_pairs_tasks.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>group_pairing_id</th>\n",
       "      <th>helperId</th>\n",
       "      <th>helperName</th>\n",
       "      <th>helpeeId</th>\n",
       "      <th>helpeeName</th>\n",
       "      <th>pairing_session_index</th>\n",
       "      <th>paired</th>\n",
       "      <th>value</th>\n",
       "      <th>userId</th>\n",
       "      <th>task</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-2Kfo9toi7yxEoB5sk</td>\n",
       "      <td>CQhpnQnrAQMZhD9tM</td>\n",
       "      <td>Nell O'Rourke</td>\n",
       "      <td>6iR9Z64HEJDcD8qbu</td>\n",
       "      <td>Matt Easterday</td>\n",
       "      <td>45</td>\n",
       "      <td>True</td>\n",
       "      <td>1.00</td>\n",
       "      <td>6iR9Z64HEJDcD8qbu</td>\n",
       "      <td>prepare talk slides on network improvement and Loft</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-2Kfo9toi7yxEoB5sk</td>\n",
       "      <td>EDEFWcagLwCfXP5Jg</td>\n",
       "      <td>Yongsung Kim</td>\n",
       "      <td>6iR9Z64HEJDcD8qbu</td>\n",
       "      <td>Matt Easterday</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>-1.00</td>\n",
       "      <td>6iR9Z64HEJDcD8qbu</td>\n",
       "      <td>prepare talk slides on network improvement and Loft</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-2Kfo9toi7yxEoB5sk</td>\n",
       "      <td>JaEySKdKKg7LAF3Yg</td>\n",
       "      <td>Eureka Foong</td>\n",
       "      <td>6iR9Z64HEJDcD8qbu</td>\n",
       "      <td>Matt Easterday</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>0.66</td>\n",
       "      <td>6iR9Z64HEJDcD8qbu</td>\n",
       "      <td>prepare talk slides on network improvement and Loft</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-2Kfo9toi7yxEoB5sk</td>\n",
       "      <td>KYnkykoMwd9fbBbWB</td>\n",
       "      <td>Julie Hui</td>\n",
       "      <td>6iR9Z64HEJDcD8qbu</td>\n",
       "      <td>Matt Easterday</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>1.00</td>\n",
       "      <td>6iR9Z64HEJDcD8qbu</td>\n",
       "      <td>prepare talk slides on network improvement and Loft</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9mdkMmj4pY8Q2TwqF-2Kfo9toi7yxEoB5sk</td>\n",
       "      <td>MJkj24zXWKhnZQCc3</td>\n",
       "      <td>Daniel George Rees Lewis</td>\n",
       "      <td>6iR9Z64HEJDcD8qbu</td>\n",
       "      <td>Matt Easterday</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>1.00</td>\n",
       "      <td>6iR9Z64HEJDcD8qbu</td>\n",
       "      <td>prepare talk slides on network improvement and Loft</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      group_pairing_id           helperId  \\\n",
       "0  9mdkMmj4pY8Q2TwqF-2Kfo9toi7yxEoB5sk  CQhpnQnrAQMZhD9tM   \n",
       "1  9mdkMmj4pY8Q2TwqF-2Kfo9toi7yxEoB5sk  EDEFWcagLwCfXP5Jg   \n",
       "2  9mdkMmj4pY8Q2TwqF-2Kfo9toi7yxEoB5sk  JaEySKdKKg7LAF3Yg   \n",
       "3  9mdkMmj4pY8Q2TwqF-2Kfo9toi7yxEoB5sk  KYnkykoMwd9fbBbWB   \n",
       "4  9mdkMmj4pY8Q2TwqF-2Kfo9toi7yxEoB5sk  MJkj24zXWKhnZQCc3   \n",
       "\n",
       "                 helperName           helpeeId      helpeeName  \\\n",
       "0  Nell O'Rourke             6iR9Z64HEJDcD8qbu  Matt Easterday   \n",
       "1  Yongsung Kim              6iR9Z64HEJDcD8qbu  Matt Easterday   \n",
       "2  Eureka Foong              6iR9Z64HEJDcD8qbu  Matt Easterday   \n",
       "3  Julie Hui                 6iR9Z64HEJDcD8qbu  Matt Easterday   \n",
       "4  Daniel George Rees Lewis  6iR9Z64HEJDcD8qbu  Matt Easterday   \n",
       "\n",
       "   pairing_session_index paired  value             userId  \\\n",
       "0  45                     True   1.00   6iR9Z64HEJDcD8qbu   \n",
       "1  45                     False -1.00   6iR9Z64HEJDcD8qbu   \n",
       "2  45                     False  0.66   6iR9Z64HEJDcD8qbu   \n",
       "3  45                     False  1.00   6iR9Z64HEJDcD8qbu   \n",
       "4  45                     False  1.00   6iR9Z64HEJDcD8qbu   \n",
       "\n",
       "                                                  task  \n",
       "0  prepare talk slides on network improvement and Loft  \n",
       "1  prepare talk slides on network improvement and Loft  \n",
       "2  prepare talk slides on network improvement and Loft  \n",
       "3  prepare talk slides on network improvement and Loft  \n",
       "4  prepare talk slides on network improvement and Loft  "
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# add helper's ability to help. keep all affinities even if people weren't actually paired (how=right; for only paired people, use how='left')\n",
    "delta_pairs_tasks_affinities = delta_pairs_tasks.merge(delta_affinities_history,\n",
    "                                                   left_on=['group_pairing_id', 'helperId', 'helpeeId', 'pairing_session_index'],\n",
    "                                                   right_on=['group_pairing_id', 'helperId', 'helpeeId', 'pairing_session_index'],\n",
    "                                                   how='right')\n",
    "\n",
    "# fill with 0 affinity if person has no partner\n",
    "delta_pairs_tasks_affinities.loc[(~delta_pairs_tasks_affinities['helpeeId'].isnull()) & (delta_pairs_tasks_affinities['value'].isnull()), 'value'] = 0\n",
    "\n",
    "# set paired to false for any unparied people (i.e., all rated affinties for people who weren't paired)\n",
    "delta_pairs_tasks_affinities.loc[delta_pairs_tasks_affinities.paired.isnull(), 'paired'] = False\n",
    "\n",
    "# fill blank names\n",
    "delta_name_dict = {row['userId']: row['name'] for index, row in delta_task_history[['userId',  'name']].iterrows()}\n",
    "delta_pairs_tasks_affinities['helperName'] = delta_pairs_tasks_affinities['helperName'].fillna(delta_pairs_tasks_affinities['helperId']).replace(delta_name_dict)\n",
    "delta_pairs_tasks_affinities['helpeeName'] = delta_pairs_tasks_affinities['helpeeName'].fillna(delta_pairs_tasks_affinities['helpeeId']).replace(delta_name_dict)\n",
    "\n",
    "# fill in blank tasks\n",
    "delta_pairs_tasks_affinities = delta_pairs_tasks_affinities.merge(delta_task_history[['group_pairing_id', 'userId', 'task']],\n",
    "                                                              left_on=['group_pairing_id', 'helpeeId'],\n",
    "                                                              right_on=['group_pairing_id', 'userId'])\n",
    "del delta_pairs_tasks_affinities['task_x']\n",
    "delta_pairs_tasks_affinities.rename(columns={'task_y': 'task'}, inplace=True)\n",
    "\n",
    "# sort table and print\n",
    "delta_pairs_tasks_affinities.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "# replace values with 1 - 5\n",
    "value_mappings = {\n",
    "    '-1.0': 1,\n",
    "    '0.0':  2,\n",
    "    '0.33': 3,\n",
    "    '0.66': 4,\n",
    "    '1.0':  5\n",
    "}\n",
    "\n",
    "delta_pairs_tasks_affinities['value'] = delta_pairs_tasks_affinities['value'].astype(str)\n",
    "delta_pairs_tasks_affinities.replace({'value': value_mappings}, inplace=True)\n",
    "\n",
    "# replace names\n",
    "name_mappings = {\n",
    "    'Leesha': 'Leesha Maliakal',\n",
    "    'Garrett': 'Garrett Hedman',\n",
    "    'eharburg@gmail.com': 'Emily Harburg'\n",
    "}\n",
    "delta_pairs_tasks_affinities.replace({'helperName': name_mappings, 'helpeeName': name_mappings}, inplace=True)\n",
    "\n",
    "# more intutive column names\n",
    "delta_pairs_tasks_affinities.rename(columns={'task': 'helpeeRequest', 'value': 'helperAbilityToHelp'}, inplace=True)\n",
    "\n",
    "# reorder columns\n",
    "delta_pairs_tasks_affinities = delta_pairs_tasks_affinities[['group_pairing_id', 'pairing_session_index', 'helperId', 'helperName',\n",
    "                                                         'helpeeId', 'helpeeName', 'helpeeRequest', 'helperAbilityToHelp', 'paired']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove nans for initial analysis, and save out csv\n",
    "delta_pairs_tasks_affinities.dropna(how='any',axis=0).to_csv('./documents/delta-pair-research-pairings.csv',\n",
    "                                                           index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summary Stats from Delta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Pairing Sessions: 213\n"
     ]
    }
   ],
   "source": [
    "# number of pairing sessions\n",
    "print('Number of Pairing Sessions: {}'.format(len(delta_pairs_history['group_pairing_id'].unique())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Unique Users: 44\n"
     ]
    }
   ],
   "source": [
    "# number of unique users\n",
    "print('Number of Unique Users: {}'.format(len(set(delta_pairs_history['helperId'].unique()).union(set(delta_pairs_history['helpeeId'].unique())))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of tasks: 1867\n"
     ]
    }
   ],
   "source": [
    "# total number of tasks\n",
    "print('Total number of tasks: {}'.format(len(delta_task_history)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    213.000000\n",
       "mean     8.765258  \n",
       "std      2.912451  \n",
       "min      1.000000  \n",
       "25%      6.000000  \n",
       "50%      9.000000  \n",
       "75%      11.000000 \n",
       "max      16.000000 \n",
       "Name: userId, dtype: float64"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# number of users/tasks per session\n",
    "delta_task_history.groupby('group_pairing_id')['userId'].count().describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of affinities: 15710\n"
     ]
    }
   ],
   "source": [
    "# total number of specified affinities\n",
    "print('Total number of affinities: {}'.format(len(delta_affinities_history)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    211.000000\n",
       "mean     74.454976 \n",
       "std      46.769359 \n",
       "min      1.000000  \n",
       "25%      30.000000 \n",
       "50%      72.000000 \n",
       "75%      103.000000\n",
       "max      234.000000\n",
       "Name: value, dtype: float64"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# number of affinities per session\n",
    "delta_affinities_history.groupby(['group_pairing_id'])['value'].count().describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    1850.000000\n",
       "mean     8.491892   \n",
       "std      2.725436   \n",
       "min      1.000000   \n",
       "25%      6.000000   \n",
       "50%      9.000000   \n",
       "75%      10.000000  \n",
       "max      15.000000  \n",
       "Name: value, dtype: float64"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# number of affinities per user per session\n",
    "delta_affinities_history.groupby(['group_pairing_id', 'helperId'])['value'].count().describe()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}