Reorganising repo. Making it flatter.

dapper.ecl

@@ -1,10 +1,10 @@
 EXPORT Bundle := MODULE(Std.BundleBase)
   EXPORT Name := 'dapper';
   EXPORT Description := 'Small macros to make your ECL more dapper. Transform tools simplify verbose operations into descriptive verbs, string tools makes matching and regex easier.';
-  EXPORT Authors := ['Rob Mansfield (rob.mansfield@proagrica.com'];
+  EXPORT Authors := ['Rob Mansfield (rob.mansfield@proagrica.com)'];
   EXPORT License := 'https://www.gnu.org/licenses/gpl-3.0.en.html';
   EXPORT Copyright := 'Copyright (C) 2018 Proagrica';
   EXPORT DependsOn := [];
-  EXPORT Version := '1.0.0';
+  EXPORT Version := '0.1.0';
   EXPORT PlatformVersion := '6.0.0';
 END;

scratchpads/IQR.ecl

@@ -0,0 +1,35 @@
+IMPORT DataScience.tools.quantile as q;
+
+EXPORT IQR := MODULE
+
+   EXPORT INrec := RECORD
+    STRING GroupVar;
+    DECIMAL32_15 num;
+  END;
+
+  EXPORT OutRec := RECORD
+    STRING GroupVar;
+    DECIMAL32_15 Q1;
+    DECIMAL32_15 Q2;
+    DECIMAL32_15 IQR;
+  END;
+
+  EXPORT IQR(DATASET(INrec) INds) := FUNCTION 
+
+    Q1 := q.quantile(inDs, 0.25);
+    Q2 := q.quantile(inDs, 0.75);
+
+    IQRange := JOIN (Q1, Q2
+                     , LEFT.GroupVar = RIGHT.GroupVar
+                     , TRANSFORM(OutRec,
+                                  SELF.GroupVar := LEFT.groupvar;
+                                  SELF.Q1 := LEFT.quant;
+                                  SELF.Q2 := RIGHT.quant;
+                                  SELF.IQR := RIGHT.quant - LEFT.quant;
+                                  )
+                     , INNER, SMART);                                
+
+    RETURN IQRange;
+    // RETURN 'BROKEN';
+  END;
+ END;

scratchpads/MedianHarvestOrDrilling.ecl

@@ -0,0 +1,98 @@
+//THIS SCRIPT IS USED TO REGENERATE THE AVERAGES USED IN THE PAST GS MODEL. IT GIVES THE MEDIAN DIFFERENCE BETWEEN THE 
+//1ST OF JAN OF CROPYEAR AND EITHER HARVEST DAY OR DRILLING DAY. IT IS QUITE A BLUNT TOOL BUT DOES THE JOB, PREFERABLE WOULD 
+//BE A SAMPLE OF A DISTRIBUTION THAT MATCHED THE DATE DISTRIBUTION. 
+#OPTION('outputlimit', 100);
+
+IMPORT Constants.InternalOutputConstants; 
+IMPORT SourceSystem;
+IMPORT OutputEntities.BaseOutputEntity;
+IMPORT datascience.Operations_Flat_Full;
+IMPORT datascience.tools.median;
+IMPORT std;
+
+outFileName := BaseOutputEntity.BuildFileName(InternalOutputConstants.DataScienceFileNamePrefix) + 'HarvestAndDrillingDifferences';
+
+////Find average drilling difference
+CZdaysDiff := PROJECT(Operations_Flat_Full.CropZoneDrills(FirstDrillingDate != ''),  
+                      TRANSFORM({STRING plannedcropname;
+                                 STRING plannedcropUID; 
+                                 INTEGER cropyear; 
+                                 STRING CropYearStart; 
+                                 INTEGER daysAfterCropYear; 
+                                 STRING FirstDrillingDate; 
+                                 // STRING CropAndYear;
+                                  },
+                                SELF.plannedcropname    := LEFT.plannedcropname;
+                                SELF.plannedcropUID     := LEFT.plannedcropUID;
+                                SELF.cropyear           := (INTEGER)LEFT.cropyear;
+                                SELF.CropYearStart      := (STRING) LEFT.cropyear + '0101';
+                                SELF.FirstDrillingDate  := LEFT.FirstDrillingDate;
+                                SELF.daysAfterCropYear := STD.Date.DaysBetween((INTEGER) SELF.CropYearStart, (INTEGER) LEFT.FirstDrillingDate);
+                                // SELF.CropAndYear        := SELF.plannedcropname + ';-;-' + (STRING) SELF.cropyear;
+                                ),
+                       LOCAL);                    
+
+medianDrilingDiffDS := PROJECT(CZdaysDiff, TRANSFORM(median.inrec, 
+                                          SELF.Group_Val := LEFT.plannedcropUID; 
+                                          SELF.num_Val   := (DECIMAL32_15) LEFT.daysAfterCropYear;
+                                          )
+                    );
+
+// OUTPUT(medianDrilingDiffDS, NAMED('medianDrilingDiffDS'));   
+medianDrillingDiff := median.median(medianDrilingDiffDS);
+
+
+////Find average harvest difference
+CZdaysDiffHarvest := PROJECT(Operations_Flat_Full.CropZoneYields(FirstHarvestDate != ''),  
+                      TRANSFORM({STRING plannedcropname;
+                                 STRING plannedcropUID; 
+                                 INTEGER cropyear; 
+                                 STRING CropYearStart; 
+                                 INTEGER daysAfterCropYear; 
+                                 STRING FirstHarvestDate; 
+                                  },
+                                SELF.plannedcropname    := LEFT.plannedcropname;
+                                SELF.plannedcropUID     := LEFT.plannedcropUID;
+                                SELF.cropyear           := (INTEGER)LEFT.cropyear;
+                                SELF.CropYearStart      := (STRING) LEFT.cropyear + '0101';
+                                SELF.FirstHarvestDate   := LEFT.FirstHarvestDate;
+                                SELF.daysAfterCropYear  := (INTEGER) STD.Date.DaysBetween((INTEGER) SELF.CropYearStart, (INTEGER) LEFT.FirstHarvestDate);
+                                )
+                       , LOCAL);                    
+
+medianDiffHarvestDS := PROJECT(CZdaysDiffHarvest, TRANSFORM(median.inrec, 
+                                          SELF.Group_Val := LEFT.plannedcropUID; 
+                                          SELF.num_Val   := (DECIMAL32_15) LEFT.daysAfterCropYear;)
+                    );
+// OUTPUT(medianDiffHarvestDS, NAMED('medianDiffHarvestDS'));   
+medianHarvestDiff := median.median(medianDiffHarvestDS);
+// OUTPUT(medianHarvestDiff, NAMED('medianHarvestDiff'));   
+
+
+differences := JOIN(medianHarvestDiff, medianDrillingDiff
+                     , LEFT.Group_Val = RIGHT.Group_Val,
+                     TRANSFORM({STRING CropEntityUID;
+                                DECIMAL32_15 HarvestDiff;
+                                DECIMAL32_15 DrillingDiff;
+                                },
+                                SELF.CropEntityUID := LEFT.Group_Val;
+                                SELF.HarvestDiff   := LEFT.median;
+                                SELF.DrillingDiff  := RIGHT.median;
+                                )
+                      , SMART);
+
+ EXPORT MedianHarvestOrDrilling := JOIN(differences, operations_flat_full.cropEntity
+                     , LEFT.CropEntityUID = RIGHT.CropEntityUID,
+                     TRANSFORM({RECORDOF(LEFT);
+                                STRING CropName;
+                                },
+                                SELF.CropName := RIGHT.name;
+                                SELF          := LEFT;
+                                )
+                      , SMART);               
+
+;
+// OUTPUT(MedianHarvestOrDrilling, NAMED('differences'));                 
+// OUTPUT(MedianHarvestOrDrilling, , outFileName, OVERWRITE, EXPIRE(30));
+
+// EXPORT MedianHarvestOrDrilling := MedianHarvestOrDrillingFile;

scratchpads/meanNoNA.ecl

@@ -0,0 +1,13 @@
+EXPORT meanNoNA(DATASET({STRING InputData}) inVals) := FUNCTION
+
+  //NOT RESISTANT TO NON-NUMERICS
+
+  RunVals := inVals(InputData != '');
+
+  a := SUM(RunVals, (DECIMAL32_12) RunVals.InputData);
+  b := COUNT(RunVals);
+
+  meanNoNA := a/b;
+
+  RETURN meanNoNA;
+  END;

scratchpads/meanNoNATests.ecl

@@ -0,0 +1,13 @@
+import datascience.Tools;
+
+tools.meanNoNA(DATASET(['1','2','3','4','','5'],
+                 {STRING InputData}));
+
+tools.meanNoNA(DATASET(['1','2','3','4'],
+                 {STRING InputData}));
+
+tools.meanNoNA(DATASET(['1','2','3','4',''],
+                 {STRING InputData}));
+
+tools.meanNoNA(DATASET(['1','2','3','4','','a'],
+                 {STRING InputData}));