Merge pull request #11 from Sotera/community_index

Community index

db_scripts/indices/index.sql

@@ -93,6 +93,14 @@ create index idx_xref_email_topic_score_email on xref_email_topic_score(email_id
 create index idx_xref_email_topic_score_email_category on xref_email_topic_score(category_id, email_id);
 create index idx_xref_email_topic_score_category on xref_email_topic_score(category_id);
 
+call drop_index_if_exists('xref_email_community', 'idx_xref_email_community_email_community');
+call drop_index_if_exists('xref_email_community', 'idx_xref_email_community_email');
+call drop_index_if_exists('xref_email_community', 'idx_xref_email_community_community');
+
+create index idx_xref_email_community_email_community on xref_email_community(email_id, community_id);
+create index idx_xref_email_community_email on xref_email_community(email_id);
+create index idx_xref_email_community_community on xref_email_community(community_id);
+
 call drop_index_if_exists('search_results', 'idx_search_results_email_id');
 create index idx_search_results_email_id on search_results(email_id);
 

db_scripts/tables/tables.sql

@@ -146,6 +146,12 @@ create table xref_email_topic_score (
    score varchar(64) not null   
 ) ENGINE=MyISAM;
 
+drop table if exists xref_email_community;
+
+create table xref_email_community (
+   email_id varchar(250) not null,
+   community_id varchar(250) not null
+) ENGINE=MyISAM;
 
 drop table if exists search_results;
 

demail/css/default.css

@@ -263,3 +263,158 @@ div.bootstrap-growl >span.close {
 .marked:hover {
     color: #ffd700;
 }
+
+
+/* circular menu */
+/**
+ modified css voodoo 
+ http://stackoverflow.com/questions/13132864/creating-a-radial-menu-in-css
+**/
+
+#radial-wrap {
+    margin: 0;
+    text-align: center;
+}
+
+#radial-wrap .email_addr {
+    font: 12px Verdana, sans-serif;
+    margin: 0 auto;
+    text-align: center;
+    display: inline-block;
+}
+
+#radial-wrap .email_addr a {
+    margin: 3px;
+    opacity: 1;
+    border-radius: 12px;
+    padding: 2px 6px;
+}
+
+.ctrl {
+    position: absolute;
+    top: 50%; left: 50%;
+    font: 1.5em/1.13 Verdana, sans-serif;
+    transition: .2s;
+}
+
+/* generic link styles */
+a.ctrl, .ctrl a {
+    display: block;
+    opacity: .56;
+    background: #333;
+    color: #fff;
+    text-align: center;
+    text-decoration: none;
+    text-shadow: 0 -1px dimgrey;
+}
+
+a.ctrl:hover, .ctrl a:hover, a.ctrl:focus, .ctrl a:focus { opacity: 1; }
+a.ctrl:focus, .ctrl a:focus { outline: none; }
+.button {
+    z-index: 2;
+    margin: -.625em;
+    width: 1.25em; height: 1.25em;
+    border-radius: 50%;
+    box-shadow: 0 0 3px 1px white;
+}
+
+.tip {
+    z-index: 1;
+    /**outline: dotted 1px white;/**/
+    margin: -5em;
+    width: 10em; height: 10em;
+    transform: scale(.001);
+    list-style: none;
+    opacity: 0;
+}
+
+/* the ends of the menu */
+/* .tip:before, .tip:after { */
+/*     position: absolute; */
+/*     top: 34.3%; */
+/*     width: .5em; height: 14%; */
+/*     opacity: .75; */
+/*     background: #333; */
+/*     content: ''; */
+/* } */
+/* .tip:before { */
+/*     left: 5.4%; */
+/*     border-radius: .25em 0 0 .25em; */
+/*     box-shadow: -1px 0 1px #333, inset 1px 0 1px #333, inset -1px 0 1px #333, */
+/*     inset 0 1px 1px #333, inset 0 -1px 1px #333; */
+/*     transform: rotate(-75deg); */
+/* } */
+/* .tip:after { */
+/*     right: 5.4%; */
+/*     border-radius: 0 .25em .25em 0; */
+/*     box-shadow: 1px 0 1px #333, inset -1px 0 1px #333, inset 1px 0 1px #333, */
+/*     inset 0 1px 1px #333, inset 0 -1px 1px #333; */
+/*     transform: rotate(75deg); */
+/* } */
+/* make the menu appear on click */
+.button:focus + .tip {
+    transform: scale(1);
+    opacity: 1;
+}
+
+.slice {
+    overflow: hidden;
+    position: absolute;
+    /**outline: dotted 1px yellow;/**/
+    width: 50%; height: 50%;
+    transform-origin: 100% 100%;
+}
+
+/* .slice:first-child { transform: rotate(-45deg) skewY(60deg); } */
+/* .slice:nth-child(2) { transform: rotate(-15deg) skewY(60deg); } */
+/* .slice:nth-child(3) { transform: rotate(15deg) skewY(60deg); } */
+/* .slice:nth-child(4) { transform: rotate(45deg) skewY(60deg); } */
+/* .slice:last-child { transform: rotate(75deg) skewY(60deg); } */
+
+.slice:first-child { transform: rotate(-15deg) skewY(60deg); }
+.slice:nth-child(2) { transform: rotate(15deg) skewY(60deg); }
+.slice:nth-child(3) { transform: rotate(45deg) skewY(60deg); }
+
+
+
+/* covers for the inner part of the links so there's no hover trigger between
+   star button & menu links; give them a red background to see them */
+.slice:after {
+    position: absolute;
+    top: 32%; left: 32%;
+    width: 136%; height: 136%;
+    border-radius: 50%;
+    /* "unskew" = skew by minus the same angle by which parent was skewed */
+    transform: skewY(-60deg);
+    content: '';
+}
+/* menu links */
+.slice a {
+    width: 200%; height: 200%;
+    border-radius: 50%;
+    box-shadow: 0 0 3px dimgrey, inset 0 0 4px white;
+    /* "unskew" & rotate by -A°/2 */
+    transform: skewY(-60deg) rotate(-15deg);
+    background: /* lateral separators */
+    linear-gradient(75deg,
+                    transparent 50%, grey 50%, transparent 54%) no-repeat 36.5% 0,
+    linear-gradient(-75deg,
+                    transparent 50%, grey 50%, transparent 54%) no-repeat 63.5% 0,
+    /* make sure inner part is transparent */
+    radial-gradient(rgba(127,127,127,0) 49%,
+                    rgba(255,255,255,.7) 51%, #333 52%);
+    background-size: 15% 15%, 15% 15%, cover;
+    line-height: 1.4;
+}
+
+/* arrow for middle link */
+.slice:nth-child(2) a:after {
+    position: absolute;
+    top: 13%; left: 50%;
+    margin: -.25em;
+    width: .5em; height: .5em;
+    box-shadow: 2px 2px 2px white;
+    transform: rotate(45deg);
+    background: linear-gradient(-45deg, #333 50%, transparent 50%);
+    content: '';
+}

demail/index.html

@@ -10,6 +10,31 @@
     <link href="css/default.css" rel="stylesheet">
   </head>
   <body>
+    <!-- hidden link //hack -->
+    <a id="alink" class='button ctrl'
+    style="height:0;width:0;position: absolute;top: -2000px;" href='#'
+    tabindex='1'></a>
+
+    <div id="radial-wrap" class="tip ctrl">
+      <div class="email_addr"><a><span></span></a></div>
+      <ul id="radial">
+        <li class='slice'>
+          <a class="clickable attach" title="Show Attachments">
+            <span class="glyphicon glyphicon-file"></span>
+          </a>
+        </li>
+        <li class='slice'>
+          <a class="clickable email" title="Search By Email">
+            <span class="glyphicon glyphicon-envelope"></span> 
+          </a>
+        </li>
+        <li class='slice'>
+          <a class="clickable community" title="Search By Community">
+            <span class="glyphicon glyphicon-search"></span> 
+          </a>
+        </li>
+      </ul>
+    </div>
 
     <!-- modal info panel -->
     <div id="info-panel" class="modal fade">

demail/js/graphtool.js

@@ -571,6 +571,10 @@ function do_search(fields, val) {
       },
       'exportable': function(args){
         return "Searching exportable emails";
+      },
+      'community': function(args){
+        var community_id = _.first(args);
+        return "Searching community";
       }
     };
     var field = _.first(varargs);
@@ -846,12 +850,43 @@ function drawGraph(graph){
       var fn = function(){
         console.log(clicks);
         if (clicks > 1){
-          do_search('email', $('#txt_search').val());
+          //do_search('email', $('#txt_search').val());
         }
         clicks=0;
       };
       if (clicks == 1){
         $('#txt_search').val(n.name);
+        var t = Math.floor($('#radial-wrap').height() / 2);
+        var l = Math.floor($('#radial-wrap').width() / 2);
+        $('#radial-wrap')
+          .css('top', (30 + d3.event.clientY - t) + "px")
+          .css('left', (d3.event.clientX - l) + "px");
+
+        $('#radial-wrap').find(".email_addr a span").first().text(n.name);
+
+        $('#radial').find(".attach").first().unbind("click")
+        .on("click", function(){
+          draw_attachments_table(n.name).done(function(){
+            $('#tab-list li:eq(4) a').tab('show');
+          });
+        });
+
+        $('#radial').find(".email").first()
+          .unbind('click')
+          .on("click", function(){
+            do_search("email", n.name);
+          }).find("span").first()
+          .css("color", colorByDomain(n.name));
+
+        $('#radial').find(".community").first()
+          .unbind('click')
+          .on("click", function(){
+          do_search("community", n.community);
+        }).find("span").first()
+          .css("color", communityColor(n.community));
+
+        _.delay(function(){  $("#alink").focus(); }, 300);
+
         _.delay(fn, 300, n.name);
       }
     };

demail/search.py

@@ -50,6 +50,13 @@
    "  where eml.exportable = 'true' "
 )
 
+stmt_node_vals_filter_community = (
+    " select distinct e.email_addr, e.community, e.community_id, e.group_id, e.total_received, e.total_sent, e.rank "
+    " from email_addr e join xref_emailaddr_email xaddr on e.email_addr = xaddr.email_addr"
+    " join xref_email_community xeml on xeml.email_id = xaddr.email_id "
+    " where xeml.community_id = %s "
+)
+
 ## Email Rows
 stmt_find_emails = (
     " select id, dir, datetime, from_addr, tos, ccs, bccs, subject, attach, bodysize "
@@ -84,6 +91,12 @@
     " from email e where exportable = 'true' "
 )
 
+stmt_find_emails_filter_community = (
+    " select distinct id, dir, datetime, from_addr, tos, ccs, bccs, subject, attach, bodysize "
+    " from email e join xref_email_community x on e.id = x.email_id "
+    " where x.community_id = %s "
+)
+
 ## all edges
 stmt_node_edges = (
     " select source, target, sum(weight)"
@@ -178,12 +191,33 @@
     "   group by source, target"
 )
 
+stmt_node_edges_filter_community = ( 
+    "  select source, target, sum(weight)"
+    " from ("
+    "    select x.`from` as source, x.recipient as target, count(1) as weight   "
+    "    from xref_recipients x join xref_email_community e on x.email_id = e.email_id "
+    "    where e.community_id = %s "
+    "    group by `from`, recipient "
+    "   union all"
+    "    select x.recipient as source, x.`from` as target, count(1) as weight   "
+    "    from xref_recipients x join xref_email_community e on x.email_id = e.email_id "
+    "    where e.community_id = %s "
+    "    group by x.`from`, x.recipient "
+    " ) as t "
+    "   group by source, target"
+)
+
 def nodeQueryObj(conn, field, args_array):
 
     #filter by exportable
     if field.lower() == "exportable": 
         return (conn, stmt_node_vals_filter_export)
 
+    #filter by community
+    if field.lower() == "community":
+        comm_id = head(args_array)
+        return (conn, stmt_node_vals_filter_community, comm_id)
+
     #filter by topic 
     if field.lower() == "topic": 
         category, idx, score = args_array[:3]
@@ -215,6 +249,11 @@ def edgeQueryObj(conn, field, args_array):
     if field.lower() == "exportable": 
         return (conn, stmt_node_edges_filter_export)
 
+    #filter by community
+    if field.lower() == "community":
+        comm_id = head(args_array)
+        return (conn, stmt_node_edges_filter_community, comm_id, comm_id)
+
     #filter by topic 
     if field.lower() == "topic": 
         category, idx, score = args_array[:3]
@@ -241,6 +280,12 @@ def emailQueryObj(conn, field, args_array):
     if field.lower() == "exportable": 
         return (conn, stmt_find_emails_filter_export)
 
+    #filter by community
+    if field.lower() == "community":
+        comm_id = head(args_array)
+        return (conn, stmt_find_emails_filter_community, comm_id)
+
+
     #filter by topic 
     if field.lower() == "topic": 
         category, idx, score = args_array[:3]

ingest/src/post_process.py

@@ -37,12 +37,19 @@
     " on x.entity_id = e.subject "
 )
 
+stmt_xref_email_community = (
+    " insert into xref_email_community (email_id, community_id) "
+    " select distinct e.id, a.community_id "
+    " from email e join xref_emailaddr_email x on e.id = x.email_id " 
+    " join email_addr a on x.email_addr = a.email_addr "
+)
+
 if __name__ == "__main__":
 
     parser = argparse.ArgumentParser(description='Post Process')
     args= parser.parse_args()
 
-    with newman_connector() as read_cnx, newman_connector() as write_cnx:
+    with newman_connector() as write_cnx:
         print "populate email_addr"
         with execute_nonquery(write_cnx.conn(), stmt_email_addr_insert) as qry:
             pass
@@ -57,3 +64,9 @@
         with execute_nonquery(write_cnx.conn(), stmt_xref_entity_email) as qry:
             pass
         write_cnx.commit()
+
+        print "populate xref_email_community"
+        with execute_nonquery(write_cnx.conn(), stmt_xref_email_community) as qry:
+            pass
+        write_cnx.commit()
+