Score_functions

Development

Author: sdrrv

README.md

@@ -130,7 +130,7 @@ redisRescore.redisUrl : "YourHostIP"
 
 # Plugin Builder
 
-## 1- First of all pull the git rep into your machine
+## 1- First pull the git rep into your machine
 <br/>
 
 ## 2- Install gradle in your machine
@@ -158,17 +158,18 @@ Once again, in order for the build to work you need to have the **Docker** insta
 
 ```bash
 $ sudo apt install docker
+$ sudo apt install docker-compose
 $ sudo docker pull redis
 ```
 <br/>
 
 ## 5- Build the Plugin
-Finally you are ready to build the Plugin.<br/>
-You can run the gradle compiler by using:
+Finally, you are ready to build the Plugin.<br/>
+You can run the gradle compiler by using in the project folder:
 ```bash
-$ gradle build
+$ ./gradlew build
 ```
-However if you are using the <strong>Intelij</strong> IDE, you can simply run the `gradle build` task.
+However, if you are using the <strong>Intelij</strong> IDE, you can simply run the `gradle build` task.
 
 <br/>
 
@@ -178,7 +179,19 @@ There will be 4 files in that dir, you want to take the `.zip` file.
 
 <br/>
 
-## 7- Install the Plugin
-See the **Plugin Installer** section
+## 7-  Install the Plugin
+Now you can `cd` into to the `bin` folder in your **ElasticSearch** instalation and run :
+```bash
+$ ./elasticsearch-plugin install file://dirOfYourPlugin
+```
+<br/>
+
+## 8- Change the Redis host
+By default, the plugin will think that your **Redis** server is running in the `localhost:6379` , that beeing said, you can change the `Host` but not the `Port`.
+The `Port` will always be `6379` because that's the default `Port` for the **Redis** server to run on. 
+<br/>
+To change the `Host`, you first need to go to the `config` folder in your **ElasticSearch** dir.
+There open the `elasticsearch.yml` file, you want to add the following:
+
 
 

build.gradle

@@ -33,6 +33,7 @@ ext {
 
 repositories {
     jcenter()
+    mavenCentral()
 }
 
 configurations.all {

src/main/java/com/bigdataboutique/elasticsearch/plugin/RedisRescoreBuilder.java

@@ -1,5 +1,6 @@
 package com.bigdataboutique.elasticsearch.plugin;
 
+import com.bigdataboutique.elasticsearch.plugin.exceptions.ScoreFunctionParseException;
 import com.bigdataboutique.elasticsearch.plugin.exceptions.ScoreOperatorException;// Exceptions
 
 import org.apache.logging.log4j.LogManager;
@@ -49,6 +50,8 @@ public class RedisRescoreBuilder extends RescorerBuilder<RedisRescoreBuilder> {
     private static final String SCORE_OPERATOR_DEFAULT = "ADD";
     private static final String BOOST_OPERATOR_DEFAULT = "ADD";
     private static final float BOOST_WEIGHT_DEFAULT = 1f;
+
+
     //------------------------------------------------------------------------------------------------------
 
     protected static final Logger log = LogManager.getLogger(RedisRescoreBuilder.class);
@@ -60,6 +63,7 @@ public class RedisRescoreBuilder extends RescorerBuilder<RedisRescoreBuilder> {
     private final String boostOperator;
     private final Float boostWeight;
     private final Float[] scoreWeights;
+    private final String[] scoreFunctions;
 
 
 
@@ -106,7 +110,8 @@ public static Float[] GetFloatArray(@Nullable ArrayList<?> arr) { // Transforms
 // Constructors--------------------------------------------------------------------------------------------------
     public RedisRescoreBuilder(final String keyField, @Nullable String keyPrefix, @Nullable String scoreOperator,
                                @Nullable String[] keyPrefixes, @Nullable String boostOperator,
-                               @Nullable Float boostWeight, @Nullable Float[] scoreWeights)
+                               @Nullable Float boostWeight, @Nullable Float[] scoreWeights,
+                               @Nullable String[] scoreFunctions)
             throws ScoreOperatorException {
 
         this.keyField =  keyField;
@@ -116,6 +121,7 @@ public RedisRescoreBuilder(final String keyField, @Nullable String keyPrefix, @N
         this.keyPrefixes = keyPrefixes;
         this.boostWeight = boostWeight == null ? BOOST_WEIGHT_DEFAULT : boostWeight;
         this.scoreWeights = scoreWeights;
+        this.scoreFunctions = scoreFunctions;
 
 
         if (!checkOperator(this.scoreOperator))
@@ -136,6 +142,8 @@ public RedisRescoreBuilder(StreamInput in) throws IOException {
         boostOperator = BOOST_OPERATOR_DEFAULT;
         boostWeight = BOOST_WEIGHT_DEFAULT;
         scoreWeights = null;
+        scoreFunctions = null;
+
 
     }
 //--------------------------------------------------------------------------------------------------
@@ -163,6 +171,8 @@ public RescorerBuilder<RedisRescoreBuilder> rewrite(QueryRewriteContext ctx) thr
     private static final ParseField BOOST_OPERATOR = new ParseField("boost_operator");
     private static final ParseField BOOST_WEIGHT = new ParseField("boost_weight");
     private static final ParseField SCORE_WEIGHTS = new ParseField("score_weights");
+    private static final ParseField SCORE_FUNCTIONS = new ParseField("score_functions");
+
 
     @Override
     protected void doXContent(XContentBuilder builder, Params params) throws IOException {
@@ -180,15 +190,20 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
 
         builder.field(SCORE_WEIGHTS.getPreferredName(), scoreWeights);
 
+        builder.field(SCORE_FUNCTIONS.getPreferredName(), scoreFunctions);
+
+
+
     }
 
+
     private static final ConstructingObjectParser<RedisRescoreBuilder, Void> PARSER =
             new ConstructingObjectParser<RedisRescoreBuilder, Void>(NAME,
             args -> {
                 try {
                     return new RedisRescoreBuilder((String) args[0], (String) args[1], (String) args[2],
                             GetStringArray((ArrayList<?>) args[3]) , (String) args[4],
-                            (Float) args[5], GetFloatArray((ArrayList<?>) args[6]));
+                            (Float) args[5], GetFloatArray((ArrayList<?>) args[6]), GetStringArray((ArrayList<?>) args[7]));
 
                 } catch (IOException e) {
                     throw new IllegalArgumentException(e);
@@ -203,6 +218,7 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
         PARSER.declareString(optionalConstructorArg(), BOOST_OPERATOR);
         PARSER.declareFloat(optionalConstructorArg(), BOOST_WEIGHT);
         PARSER.declareFloatArray(optionalConstructorArg(), SCORE_WEIGHTS);
+        PARSER.declareStringArray(optionalConstructorArg(),SCORE_FUNCTIONS);
     }
     public static RedisRescoreBuilder fromXContent(XContentParser parser) {
         return PARSER.apply(parser, null);
@@ -213,7 +229,7 @@ public RescoreContext innerBuildContext(int windowSize, QueryShardContext contex
         IndexFieldData<?> keyField =
                 this.keyField == null ? null : context.getForField(context.fieldMapper(this.keyField));
         return new RedisRescoreContext(windowSize, keyPrefix, keyField, scoreOperator,
-                keyPrefixes, boostOperator, boostWeight, scoreWeights);
+                keyPrefixes, boostOperator, boostWeight, scoreWeights, scoreFunctions);
     }
 
     @Override
@@ -267,18 +283,25 @@ Float[] scoreWeights(){
         return scoreWeights;
     }
 
+    @Nullable
+    String[] scoreFunctions(){
+        return scoreFunctions;
+    }
+
+
     private static class RedisRescoreContext extends RescoreContext {
         private final String keyPrefix;
         private final String[] keyPrefixes;
         private final String scoreOperator;
         private final String boostOperator;
         private final Float boostWeight;
         private final Float[] scoreWeights;
+        private final String[] scoreFunctions;
         @Nullable
         private final IndexFieldData<?> keyField;
 
         RedisRescoreContext(int windowSize, String keyPrefix, @Nullable IndexFieldData<?> keyField, String scoreOperator,
-                            String[] keyPrefixes, String boostOperator, Float boostWeight, Float[] scoreWeights) {
+                            String[] keyPrefixes, String boostOperator, Float boostWeight, Float[] scoreWeights, String[] scoreFunctions) {
             super(windowSize, RedisRescorer.INSTANCE);
             this.keyPrefix = keyPrefix;
             this.keyField = keyField;
@@ -287,6 +310,7 @@ private static class RedisRescoreContext extends RescoreContext {
             this.boostOperator = boostOperator;
             this.boostWeight = boostWeight;
             this.scoreWeights = scoreWeights;
+            this.scoreFunctions = scoreFunctions;
 
         }
     }
@@ -382,21 +406,25 @@ public TopDocs rescore(TopDocs topDocs, IndexSearcher searcher, RescoreContext r
                                     float scoreWeight = getScoreWeight(context.scoreWeights, j);
 
                                     if (redisScore == 0)
-                                        redisScore = getScoreFactor(term, prefix, scoreWeight);
+                                        redisScore = getRescore(term, prefix, scoreWeight, j, context.scoreFunctions);
 
                                     else {
                                         switch (context.scoreOperator) {
                                             case "ADD":
-                                                redisScore += getScoreFactor(term, prefix, scoreWeight);
+                                                redisScore += getRescore(term, prefix, scoreWeight,
+                                                        j, context.scoreFunctions);
                                                 break;
                                             case "MULTIPLY":
-                                                redisScore *= getScoreFactor(term, prefix, scoreWeight);
+                                                redisScore *= getRescore(term, prefix, scoreWeight,
+                                                        j, context.scoreFunctions);
                                                 break;
                                             case "SUBTRACT":
-                                                redisScore -= getScoreFactor(term, prefix, scoreWeight);
+                                                redisScore -= getRescore(term, prefix, scoreWeight,
+                                                        j, context.scoreFunctions);
                                                 break;
                                             case "SET":
-                                                redisScore = getScoreFactor(term, prefix, scoreWeight);
+                                                redisScore = getRescore(term, prefix, scoreWeight,
+                                                        j, context.scoreFunctions);
                                                 break;
                                         }
                                     }
@@ -426,25 +454,25 @@ else if (numericDocValues != null) {
                                 float scoreWeight = getScoreWeight(context.scoreWeights, j);
 
                                 if (redisScore == 0)
-                                    redisScore = getScoreFactor(String.valueOf(numericDocValues.nextValue()),
-                                            prefix, scoreWeight);
+                                    redisScore = getRescore(String.valueOf(numericDocValues.nextValue()),
+                                            prefix, scoreWeight, j, context.scoreFunctions);
                                 else {
                                     switch (context.scoreOperator) {
                                         case "ADD":
-                                            redisScore += getScoreFactor(String.valueOf(numericDocValues.nextValue()),
-                                                    prefix, scoreWeight);
+                                            redisScore += getRescore(String.valueOf(numericDocValues.nextValue()),
+                                                    prefix, scoreWeight, j, context.scoreFunctions);
                                             break;
                                         case "MULTIPLY":
-                                            redisScore *= getScoreFactor(String.valueOf(numericDocValues.nextValue()),
-                                                    prefix, scoreWeight);
+                                            redisScore *= getRescore(String.valueOf(numericDocValues.nextValue()),
+                                                    prefix, scoreWeight, j, context.scoreFunctions);
                                             break;
                                         case "SUBTRACT":
-                                            redisScore -= getScoreFactor(String.valueOf(numericDocValues.nextValue()),
-                                                    prefix, scoreWeight);
+                                            redisScore -= getRescore(String.valueOf(numericDocValues.nextValue()),
+                                                    prefix, scoreWeight, j, context.scoreFunctions);
                                             break;
                                         case "SET":
-                                            redisScore = getScoreFactor(String.valueOf(numericDocValues.nextValue()),
-                                                    prefix, scoreWeight);
+                                            redisScore = getRescore(String.valueOf(numericDocValues.nextValue()),
+                                                    prefix, scoreWeight, j, context.scoreFunctions);
                                             break;
                                     }
                                 }
@@ -490,6 +518,41 @@ else if (numericDocValues != null) {
             });
             return topDocs;
         }
+
+        private static float getRescore(final String key, @Nullable final String keyPrefix, @Nullable float scoreWeight,
+                                        @Nullable int keyPrefixesIndex, @Nullable String[] scoreFunctions)
+                throws ScoreFunctionParseException {
+
+            if (scoreFunctions == null || scoreFunctions.length == 0)
+                return getScoreFactor(key, keyPrefix, scoreWeight);
+            //add exception here
+            float scoreFactor = getScoreFactor(key, keyPrefix,1);
+            Float res = null;
+
+            if( keyPrefixesIndex >= scoreFunctions.length || Objects.equals(scoreFunctions[keyPrefixesIndex], "null")){
+                return scoreFactor * scoreWeight;
+            }
+
+            try {
+                String[] parsed = ScoreFunctionParser.getScoreFunctionParser().parse(scoreFunctions[keyPrefixesIndex],
+                        String.valueOf(scoreFactor));
+
+
+                //--------------------------------------------Functions---------------------------------------------------
+                switch (parsed[0]) {
+                    case "pow":
+                        res = ScoreFunctionsObj.get().pow(Float.parseFloat(parsed[1]), Float.parseFloat(parsed[2]));
+                        break;
+                }
+                //--------------------------------------------------------------------------------------------------------
+            } catch(Exception e){
+                throw new ScoreFunctionParseException(scoreFunctions[keyPrefixesIndex]);
+            }
+
+            return res == null ? scoreFactor * scoreWeight : res * scoreWeight ;
+        }
+
+
         //ScoreWeight Default Value must be 1
         private static float getScoreFactor(final String key, @Nullable final String keyPrefix,float scoreWeight) {
             assert key != null;
@@ -502,8 +565,9 @@ private static float getScoreFactor(final String key, @Nullable final String key
                     return 1.0f;
                 }
 
-                try {
+                try { // Here
                     return Float.parseFloat(factor) * scoreWeight;
+
                 } catch (NumberFormatException ignored_e) {
                     log.warn("Redis rescore factor NumberFormatException for key " + fullKey);
                     return 1.0f;

src/main/java/com/bigdataboutique/elasticsearch/plugin/ScoreFunctionParser.java

@@ -0,0 +1,45 @@
+package com.bigdataboutique.elasticsearch.plugin;
+import java.util.ArrayList;
+import java.util.List;
+
+public class ScoreFunctionParser {
+    private static ScoreFunctionParser _scoreFunctionParser;
+    private ScoreFunctionParser(){}
+
+    public static ScoreFunctionParser getScoreFunctionParser(){
+        if(_scoreFunctionParser == null)
+            _scoreFunctionParser = new ScoreFunctionParser();
+        return _scoreFunctionParser;
+    }
+
+    public String[] parse(String toParse, String replacer){
+        List<String> res = new ArrayList<>();
+        String function;
+        int i = 0;
+        StringBuilder builder = new StringBuilder();
+
+        while (toParse.charAt(i) != '(') { // GET THE FUNCTION
+            builder.append(toParse.charAt(i));
+            i++;
+        }
+        res.add(builder.toString());
+        builder.setLength(0);
+        i++;
+        while (i < toParse.length()){ // params
+            char current = toParse.charAt(i);
+            if (current == ',' || current == ')'){
+                res.add(builder.toString());
+                builder.setLength(0);
+            }
+            else if (current == '_'){
+                res.add(replacer);
+                builder.setLength(0);
+                i++;
+            }
+            else
+                builder.append(current);
+            i++;
+        }
+        return res.toArray(new String[0]);
+    }
+}

src/main/java/com/bigdataboutique/elasticsearch/plugin/ScoreFunctionsObj.java

@@ -0,0 +1,19 @@
+package com.bigdataboutique.elasticsearch.plugin;
+//import java.lang.Math;
+
+public class ScoreFunctionsObj {
+    private static ScoreFunctionsObj _scoreFunctions;
+    private ScoreFunctionsObj(){}
+
+    public static ScoreFunctionsObj get(){
+        if (_scoreFunctions == null )
+            _scoreFunctions = new ScoreFunctionsObj();
+        return _scoreFunctions;
+    }
+
+    //----------------------Functions-------------------------------------
+    public float pow(float base, float exponent){
+        return (float) Math.pow(base,exponent);
+    }
+    //-------------------------------------------------------------
+}

src/main/java/com/bigdataboutique/elasticsearch/plugin/exceptions/ScoreFunctionParseException.java

@@ -0,0 +1,8 @@
+package com.bigdataboutique.elasticsearch.plugin.exceptions;
+import java.io.IOException;
+
+public class ScoreFunctionParseException extends IOException{
+    public ScoreFunctionParseException(String func){
+        super("Bad function body: "+ func);
+    }
+}