plumb configuration for qw-fs max hops, made depth predicate class

pom.xml

@@ -96,11 +96,21 @@
             <artifactId>log4j-api</artifactId>
             <version>${log4j2.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-1.2-api</artifactId>
+            <version>${log4j2.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.logging.log4j</groupId>
             <artifactId>log4j-core</artifactId>
             <version>${log4j2.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-log4j12</artifactId>
+            <version>1.7.12</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.lucene</groupId>
             <artifactId>lucene-core</artifactId>
@@ -337,7 +347,7 @@
                                     <includes>
                                         <include>**/GraphDatabaseBenchmarkTest.java</include>
                                     </includes>
-                                    <argLine>-Xmx32g</argLine>
+                                    <argLine>-Xmx32g -ea</argLine>
                                     <skip>false</skip>
                                     <systemPropertyVariables>
                                         <log4j.configurationFile>${basedir}/src/test/resources/META-INF/log4j2.xml</log4j.configurationFile>

src/main/java/eu/socialsensor/benchmarks/PermutingBenchmarkBase.java

@@ -55,7 +55,7 @@ public void startBenchmarkInternal()
             startBenchmarkInternalOnePermutation(bench.getSelectedDatabases(), 1);
         }
 
-        LOG.info(String.format("%s Benchmark finished", type.longname()));
+        LOG.info(String.format("%s Benchmark Finished", type.longname()));
         post();
     }
 

src/main/java/eu/socialsensor/dataset/Dataset.java

@@ -33,7 +33,7 @@ public Dataset(File datasetFile, Random random, int randomNodeSetSize)
 
         //shuffle
         final List<Integer> nodeList = new ArrayList<>(nodes);
-        Collections.shuffle(nodeList);
+        Collections.shuffle(nodeList, random);
 
         //choose randomNodeSetSize of them
         generatedNodes = new ArrayList<Integer>(randomNodeSetSize);

src/main/java/eu/socialsensor/graphdatabases/GraphDatabaseBase.java

@@ -5,6 +5,7 @@
 import java.util.List;
 import java.util.Set;
 
+import org.apache.commons.logging.Log;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -33,8 +34,10 @@ public abstract class GraphDatabaseBase<VertexIteratorType, EdgeIteratorType, Ve
     private final Timer getAllEdgesTimes;
     private final Timer shortestPathTimes;
     private final List<Integer> randomNodes;
+    protected final int maxHops;
 
-    protected GraphDatabaseBase(GraphDatabaseType type, File dbStorageDirectory, List<Integer> randomNodes)
+    protected GraphDatabaseBase(GraphDatabaseType type, File dbStorageDirectory, List<Integer> randomNodes,
+                                int shortestPathMaxHops)
     {
         this.type = type;
         final String queryTypeContext = type.getShortname() + QUERY_CONTEXT;
@@ -45,6 +48,7 @@ protected GraphDatabaseBase(GraphDatabaseType type, File dbStorageDirectory, Lis
         this.getAllEdgesTimes = GraphDatabaseBenchmark.metrics.timer(queryTypeContext + "getAllEdges");
         this.shortestPathTimes = GraphDatabaseBenchmark.metrics.timer(queryTypeContext + "shortestPath");
         this.randomNodes = randomNodes;
+        this.maxHops = shortestPathMaxHops;
 
         this.dbStorageDirectory = dbStorageDirectory;
         if (!this.dbStorageDirectory.exists())

src/main/java/eu/socialsensor/graphdatabases/Neo4jGraphDatabase.java

@@ -62,9 +62,9 @@ public static enum RelTypes implements RelationshipType
 
     public static Label NODE_LABEL = DynamicLabel.label("Node");
 
-    public Neo4jGraphDatabase(File dbStorageDirectoryIn, boolean batchLoading, List<Integer> randomNodes)
+    public Neo4jGraphDatabase(File dbStorageDirectoryIn, boolean batchLoading, List<Integer> randomNodes, int shortestPathMaxHops)
     {
-        super(GraphDatabaseType.NEO4J, dbStorageDirectoryIn, randomNodes);
+        super(GraphDatabaseType.NEO4J, dbStorageDirectoryIn, randomNodes, shortestPathMaxHops);
 
         if(batchLoading) {
             neo4jGraph = null;
@@ -207,7 +207,7 @@ public void findAllNodeNeighbours() {
     public void shortestPath(Node n1, Integer i)
     {
         PathFinder<Path> finder
-            = GraphAlgoFactory.shortestPath(PathExpanders.forType(Neo4jGraphDatabase.RelTypes.SIMILAR), 5);
+            = GraphAlgoFactory.shortestPath(PathExpanders.forType(Neo4jGraphDatabase.RelTypes.SIMILAR), maxHops);
         Node n2 = getVertex(i);
         Path path = finder.findSinglePath(n1, n2);
 

src/main/java/eu/socialsensor/graphdatabases/OrientGraphDatabase.java

@@ -49,7 +49,8 @@ public class OrientGraphDatabase extends GraphDatabaseBase<Iterator<Vertex>, Ite
     @SuppressWarnings("deprecation")
     public OrientGraphDatabase(BenchmarkConfiguration config, File dbStorageDirectoryIn)
     {
-        super(GraphDatabaseType.ORIENT_DB, dbStorageDirectoryIn, config.getRandomNodeList());
+        super(GraphDatabaseType.ORIENT_DB, dbStorageDirectoryIn, config.getRandomNodeList(),
+                config.getShortestPathMaxHops());
         OGlobalConfiguration.STORAGE_COMPRESSION_METHOD.setValue("nothing");
         OGlobalConfiguration.STORAGE_KEEP_OPEN.setValue(false);
         graph = getGraph(dbStorageDirectory);
@@ -102,9 +103,8 @@ public void shortestPath(final Vertex v1, Integer i)
         @SuppressWarnings("unused")
         final OrientVertex v2 = (OrientVertex) getVertex(i);
 
-        //TODO(amcp) need to do something about the number 5
 //        List<ORID> result = (List<ORID>) new OSQLFunctionShortestPath().execute(graph,
-//            null, null, new Object[] { ((OrientVertex) v1).getRecord(), v2.getRecord(), Direction.OUT, 5 },
+//            null, null, new Object[] { ((OrientVertex) v1).getRecord(), v2.getRecord(), Direction.OUT, maxHops },
 //            new OBasicCommandContext());
 //
 //        result.size();

src/main/java/eu/socialsensor/graphdatabases/TitanGraphDatabase.java

@@ -10,13 +10,15 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
 
 import com.google.common.base.Stopwatch;
 import org.apache.commons.configuration.Configuration;
 import org.apache.commons.configuration.MapConfiguration;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.tinkerpop.gremlin.process.traversal.Path;
+import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
@@ -64,17 +66,14 @@
 public class TitanGraphDatabase extends GraphDatabaseBase<Iterator<Vertex>, Iterator<Edge>, Vertex, Edge>
 {
     private static final Logger LOG = LogManager.getLogger();
-    public static final String INSERTION_TIMES_OUTPUT_PATH = "data/titan.insertion.times";
-
-    double totalWeight;
 
     private final StandardTitanGraph graph;
     private final BenchmarkConfiguration config;
 
     public TitanGraphDatabase(GraphDatabaseType type, BenchmarkConfiguration config, File dbStorageDirectory,
             boolean batchLoading)
     {
-        super(type, dbStorageDirectory, config.getRandomNodeList());
+        super(type, dbStorageDirectory, config.getRandomNodeList(), config.getShortestPathMaxHops());
         this.config = config;
         if (!GraphDatabaseType.TITAN_FLAVORS.contains(type))
         {
@@ -289,6 +288,19 @@ public void shutdownMassiveGraph()
         shutdown();
     }
 
+    public class DepthPredicate implements Predicate<Traverser<T>> {
+        private final int hops;
+        public DepthPredicate(int hops) {
+            this.hops = hops;
+        }
+
+        @Override
+        public boolean test(Traverser<T> it) {
+            LOG.trace("testing {}", it.path());
+            return it.path().size() <= hops;
+        }
+    }
+
     @Override
     public void shortestPath(final Vertex fromNode, Integer targetNode)
     {
@@ -299,19 +311,17 @@ public void shortestPath(final Vertex fromNode, Integer targetNode)
         //            until you map to the target toNode and the path is six vertices long or less
         //            only return one path
 //g.V().has("nodeId", 775).repeat(out('similar').simplePath()).until(has('nodeId', 990).and().filter {it.path().size() <= 5}).limit(1).path().by('nodeId')
-        GraphTraversal<?, Path> t =
-        g.V().has(NODE_ID, fromNode.<Integer>value(NODE_ID))
+        final DepthPredicate maxDepth = new DepthPredicate(maxHops);
+        final Integer fromNodeId = fromNode.<Integer>value(NODE_ID);
+        LOG.trace("finding path from {} to {} max hops {}", fromNodeId, targetNode, maxHops);
+        final GraphTraversal<?, Path> t =
+        g.V().has(NODE_ID, fromNodeId)
                 .repeat(
                         __.out(SIMILAR)
                                 .simplePath())
                 .until(
                         __.has(NODE_ID, targetNode)
-                        .and(
-                                __.filter(it -> {
-//when the size of the path in the traverser object is six, that means this traverser made 4 hops from the
-//fromNode, a total of 5 vertices
-                                    return it.path().size() <= 5;
-                                }))
+                        .and(__.filter( maxDepth ))
                 )
                 .limit(1)
                 .path();
@@ -321,8 +331,8 @@ public void shortestPath(final Vertex fromNode, Integer targetNode)
                     final int pathSize = it.size();
                     final long elapsed = watch.elapsed(TimeUnit.MILLISECONDS);
                     watch.stop();
-                    if(elapsed > 200) { //threshold for debugging
-                        LOG.info("from @ " + fromNode.value(NODE_ID) +
+                    if(elapsed > 500) { //threshold for debugging
+                        LOG.warn("from @ " + fromNode.value(NODE_ID) +
                                 " to @ " + targetNode.toString() +
                                 " took " + elapsed + " ms, " + pathSize + ": " + it.toString());
                     }

src/main/java/eu/socialsensor/insert/InsertionBase.java

@@ -121,7 +121,7 @@ public final void createGraph(File datasetFile, int scenarioNumber)
             }
         });
         post();
-        logger.info("Edges: " + i.get());
+        logger.trace("Edges: " + i.get());
         insertionTimes.add((double) watch.elapsed(TimeUnit.MILLISECONDS));
 
         if (single)

src/main/java/eu/socialsensor/insert/TitanMassiveInsertion.java

@@ -50,7 +50,7 @@ public void relateNodes(Vertex src, Vertex dest)
 
     @Override
     protected void post() {
-        logger.info("vertices: " + vertexCache.size());
+        logger.trace("vertices: " + vertexCache.size());
         tx.commit(); //mutation work is done here
         Preconditions.checkState(graph.getOpenTransactions().isEmpty());
     }

src/main/java/eu/socialsensor/main/BenchmarkConfiguration.java

@@ -54,6 +54,7 @@ public class BenchmarkConfiguration
     private static final String PERMUTE_BENCHMARKS = "permute-benchmarks";
     private static final String RANDOM_NODES = "shortest-path-random-nodes";
     private static final String RANDOM_SEED = "random-seed";
+    private static final String MAX_HOPS = "shortest-path-max-hops";
 
     private static final Set<String> metricsReporters = new HashSet<String>();
     static {
@@ -103,6 +104,7 @@ public class BenchmarkConfiguration
     private final String dynamodbTablePrefix;
     private final boolean customIds;
     private final long tuplMinCacheSize;
+    private final int shortestPathMaxHops;
 
     private final Random random;
 
@@ -179,6 +181,7 @@ public BenchmarkConfiguration(Configuration appconfig)
         // load the dataset
         random = new Random(socialsensor.getInt(RANDOM_SEED, 17 /*default*/));
         numShortestPathRandomNodes = socialsensor.getInteger(RANDOM_NODES, new Integer(101));
+        shortestPathMaxHops = socialsensor.getInteger(MAX_HOPS, 5);
         DatasetFactory.getInstance().createAndGetDataset(dataset, random, numShortestPathRandomNodes);
 
         if (!socialsensor.containsKey(PERMUTE_BENCHMARKS))
@@ -449,4 +452,8 @@ public Random getRandom() {
     public List<Integer> getRandomNodeList() {
         return DatasetFactory.getInstance().getDataset(this.dataset).getRandomNodes();
     }
+
+    public int getShortestPathMaxHops() {
+        return shortestPathMaxHops;
+    }
 }

src/main/java/eu/socialsensor/utils/Utils.java

@@ -191,7 +191,8 @@ public static final GraphDatabase<?,?,?,?> createDatabaseInstance(BenchmarkConfi
         }
         else if (GraphDatabaseType.NEO4J == type)
         {
-            graphDatabase = new Neo4jGraphDatabase(dbStorageDirectory, batchLoading, config.getRandomNodeList());
+            graphDatabase = new Neo4jGraphDatabase(dbStorageDirectory, batchLoading, config.getRandomNodeList(),
+                    config.getShortestPathMaxHops());
         }
         else if (GraphDatabaseType.ORIENT_DB == type)
         {

src/test/java/eu/socialsensor/main/GraphDatabaseBenchmarkTest.java

@@ -9,6 +9,7 @@ public class GraphDatabaseBenchmarkTest
     public void testGraphDatabaseBenchmark()
     {
         GraphDatabaseBenchmark bench = new GraphDatabaseBenchmark(null /* inputPath */);
+
         try
         {
             bench.run();

src/test/resources/META-INF/input.properties

@@ -1,6 +1,6 @@
 # Choose which data sets you want to include in the benchmark by removing the contents.
 #Enron
-eu.socialsensor.dataset=data/Email-Enron.txt
+#eu.socialsensor.dataset=data/Email-Enron.txt
 #Amazon
 #eu.socialsensor.dataset=data/Amazon0601.txt
 #YouTube
@@ -11,7 +11,7 @@ eu.socialsensor.dataset=data/Email-Enron.txt
 #eu.socialsensor.actual-communities=com-lj.all.cmty.txt
 #Synthetic
 #can change the number in the filename of the synthetic datasets to 1000, 5000, 10000, 20000, 30000, 40000, 50000
-#eu.socialsensor.dataset=data/network10000.dat
+eu.socialsensor.dataset=data/network5000.dat
 #eu.socialsensor.actual-communities=data/community50000.dat
 
 eu.socialsensor.database-storage-directory=storage
@@ -24,7 +24,7 @@ eu.socialsensor.metrics.csv.directory=metrics
 
 # Choose which databases you want to in the benchmark by removing the comments.
 # Available dbs are:
-eu.socialsensor.databases=tbdb
+#eu.socialsensor.databases=tbdb
 eu.socialsensor.databases=ttupl
 #eu.socialsensor.databases=tddb
 #eu.socialsensor.databases=tc
@@ -40,7 +40,8 @@ eu.socialsensor.titan.custom-ids=true
 # page-size - Number of results to pull when iterating over a storage backend (default 100)
 eu.socialsensor.titan.page-size=100
 # to disable buffering on mutations, set to zero. Default 1024. This will set the queue size as well
-eu.socialsensor.titan.buffer-size=10000
+# use max int as buffer size
+eu.socialsensor.titan.buffer-size=2147483647
 # id block size default 10000
 eu.socialsensor.titan.ids.block-size=10000
 # Titan DynamoDB options
@@ -82,8 +83,9 @@ eu.socialsensor.benchmarks=MASSIVE_INSERTION
 #eu.socialsensor.benchmarks=SINGLE_INSERTION
 eu.socialsensor.benchmarks=FIND_NEIGHBOURS
 eu.socialsensor.benchmarks=FIND_ADJACENT_NODES
-#eu.socialsensor.benchmarks=FIND_SHORTEST_PATH
-eu.socialsensor.shortest-path-random-nodes=100
+eu.socialsensor.benchmarks=FIND_SHORTEST_PATH
+eu.socialsensor.shortest-path-random-nodes=101
+eu.socialsensor.shortest-path-max-hops=5
 
 # The clustering benchmark is not permutable even if eu.socialsensor.permute-benchmarks=true
 #eu.socialsensor.benchmarks=CLUSTERING

src/test/resources/META-INF/log4j2.xml

@@ -1,13 +1,17 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<Configuration status="fatal">
-  <Appenders>
-    <Console name="Console" target="SYSTEM_OUT">
-      <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %c{1} - %msg%n"/>
-    </Console>
-  </Appenders>
-  <Loggers>
-    <Root level="info">
-      <AppenderRef ref="Console"/>
-    </Root>
-  </Loggers>
+<Configuration status="warn">
+    <Appenders>
+        <Console name="Console" target="SYSTEM_OUT">
+            <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %c{1} - %msg%n"/>
+        </Console>
+    </Appenders>
+    <Loggers>
+        <Logger name="com.thinkaurelius.titan.diskstorage.berkeleyje.BerkeleyJEKeyValueStore" level="ERROR"/>
+        <Logger name="com.thinkaurelius.titan.diskstorage.log.kcvs.KCVSLog" level="WARN"/>
+        <Logger name="com.thinkaurelius.titan.diskstorage.Backend" level="WARN"/>
+        <Logger name="com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration" level="WARN"/>
+        <Root level="info">
+            <AppenderRef ref="Console"/>
+        </Root>
+    </Loggers>
 </Configuration>