fix merging shard data

textexploration · Apr 2, 2020 · 000e4f3 · 000e4f3
1 parent 4da90f8
commit 000e4f3
Show file tree

Hide file tree

Showing 12 changed files with 392 additions and 205 deletions.
diff --git a/conf/solr/schemaMiddelnederlands.xml b/conf/solr/schemaMiddelnederlands.xml
@@ -56,11 +56,11 @@
 	<!-- location -->	
 	<field name="localizationKloekecode" type="middelnederlands_string"
 		multiValued="false" indexed="true" stored="true" />
-	<field name="localizationTown" type="middelnederlands_text" 
+	<field name="localizationTown" type="middelnederlands_string" 
 	    multiValued="false" indexed="true" stored="true" />
-	<field name="localizationProvince" type="middelnederlands_text" 
+	<field name="localizationProvince" type="middelnederlands_string" 
 	    multiValued="false" indexed="true" stored="true" />
-	<field name="localizationCountry" type="middelnederlands_text" 
+	<field name="localizationCountry" type="middelnederlands_string" 
 	    multiValued="false" indexed="true" stored="true" />
 	<field name="localizationLatLng" type="middelnederlands_geo" 
 	    multiValued="false" indexed="true" stored="true" />   

diff --git a/src/main/java/mtas/codec/util/CodecCollector.java b/src/main/java/mtas/codec/util/CodecCollector.java
diff --git a/src/main/java/mtas/codec/util/heatmap/PrefixTreeMtasCounter.java b/src/main/java/mtas/codec/util/heatmap/PrefixTreeMtasCounter.java
@@ -181,13 +181,16 @@ private CellValues computeValuesAtThisTerm() throws IOException {
                   for(int i=0; i<heatmap.hm.functions.size(); i++) {
                     SubComponentFunction f = heatmap.hm.functions.get(i);
                     try{
+                      long[] argsD = args[docSetCounter].clone();
+                      for(int k=0; k<argsD.length; k++) {
+                        argsD[k] = (argsD[k]>0)?1:0;
+                      }
                       if(f.dataType.equals(CodecUtil.DATA_TYPE_LONG)) {
-
-                        functionValueLong = f.parserFunction.getValueLong(args[docSetCounter], positions[docSetCounter]); 
+                        functionValueLong = f.parserFunction.getValueLong(args[docSetCounter], argsD, positions[docSetCounter], 1); 
                         functionValuesLong[i][functionValuesCounter[i]] = functionValueLong;
                         functionValuesCounter[i]++;
                       } else if(f.dataType.equals(CodecUtil.DATA_TYPE_DOUBLE)) {                      
-                        functionValueDouble = f.parserFunction.getValueDouble(args[docSetCounter], positions[docSetCounter]); 
+                        functionValueDouble = f.parserFunction.getValueDouble(args[docSetCounter], argsD, positions[docSetCounter], 1); 
                         functionValuesDouble[i][functionValuesCounter[i]] = functionValueDouble;
                         functionValuesCounter[i]++;
                       }

diff --git a/src/main/java/mtas/parser/function/MtasFunctionParser.jj b/src/main/java/mtas/parser/function/MtasFunctionParser.jj
@@ -36,7 +36,12 @@ SKIP :
 
 TOKEN :
 {
-  < VARIABLE : "$q" ([ "0"-"9" ])+ >
+  < VARIABLE_Q : "$q" ([ "0"-"9" ])+ >
+}
+
+TOKEN :
+{
+  < VARIABLE_D : "$d" ([ "0"-"9" ])* >
 }
 
 TOKEN :
@@ -201,12 +206,24 @@ private MtasFunctionParserItem parserFunctionItem() throws ParseException :
 }
 {
   (
-    v = < VARIABLE >
+    v = < VARIABLE_Q >
     {
       vInt = Integer.parseInt(v.image.substring(2));
-      item = new MtasFunctionParserItem(MtasFunctionParserItem.TYPE_ARGUMENT, vInt);
+      item = new MtasFunctionParserItem(MtasFunctionParserItem.TYPE_ARGUMENT_Q, vInt);
       return item;
     }
+  |
+    v = < VARIABLE_D >
+    {
+      if(v.image.length() >2) { 
+        vInt = Integer.parseInt(v.image.substring(2));
+        item = new MtasFunctionParserItem(MtasFunctionParserItem.TYPE_ARGUMENT_D, vInt);
+        return item;
+	  } else {
+	    item = new MtasFunctionParserItem(MtasFunctionParserItem.TYPE_D);
+	    return item;
+	  }  
+    }
   | 
     < N >
     {

diff --git a/src/main/java/mtas/parser/function/util/MtasFunctionParserFunction.java b/src/main/java/mtas/parser/function/util/MtasFunctionParserFunction.java
@@ -54,20 +54,19 @@ public abstract class MtasFunctionParserFunction {
    * @param n the n
    * @return the response
    */
-  public final MtasFunctionParserFunctionResponse getResponse(long[] args,
-      long n) {
+  public final MtasFunctionParserFunctionResponse getResponse(long[] argsQ, long[] argsD, long n, long d) {
     if (dataType.equals(CodecUtil.DATA_TYPE_LONG)) {
       try {
-        long l = getValueLong(args, n);
-        return new MtasFunctionParserFunctionResponseLong(l, true);
+        long lv = getValueLong(argsQ, argsD, n, d);
+        return new MtasFunctionParserFunctionResponseLong(lv, true);
       } catch (IOException e) {
         log.debug(e);
         return new MtasFunctionParserFunctionResponseLong(0, false);
       }
     } else if (dataType.equals(CodecUtil.DATA_TYPE_DOUBLE)) {
       try {
-        double d = getValueDouble(args, n);
-        return new MtasFunctionParserFunctionResponseDouble(d, true);
+        double dv = getValueDouble(argsQ, argsD, n, d); 
+        return new MtasFunctionParserFunctionResponseDouble(dv, true);
       } catch (IOException e) {
         log.debug(e);
         return new MtasFunctionParserFunctionResponseDouble(0, false);
@@ -85,7 +84,7 @@ public final MtasFunctionParserFunctionResponse getResponse(long[] args,
    * @return the value double
    * @throws IOException Signals that an I/O exception has occurred.
    */
-  public abstract double getValueDouble(long[] args, long n) throws IOException;
+  public abstract double getValueDouble(long[] argsQ, long[] argsD, long n, long d) throws IOException;
 
   /**
    * Gets the value long.
@@ -95,7 +94,7 @@ public final MtasFunctionParserFunctionResponse getResponse(long[] args,
    * @return the value long
    * @throws IOException Signals that an I/O exception has occurred.
    */
-  public abstract long getValueLong(long[] args, long n) throws IOException;
+  public abstract long getValueLong(long[] argsQ, long[] argsD, long n, long d) throws IOException;
 
   /**
    * Close.