Twister2 Deep Learning (DO NOT MERGE)

scripts/package/BUILD

@@ -44,6 +44,7 @@ load("//tools/rules:twister2_core.bzl", "twister2_storm_files")
 load("//tools/rules:twister2_core.bzl", "twister2_beam_files")
 load("//tools/rules:twister2_core.bzl", "twister2_core_checkpointing_files")
 load("//tools/rules:twister2_core.bzl", "twister2_core_tset_files")
+load("//tools/rules:twister2_core.bzl", "twister2_core_dl_files")
 load("//tools/rules:twister2_core.bzl", "twister2_python_support_files")
 load("//tools/rules:twister2_master.bzl", "twister2_master_lib_files")
 load("//tools/rules:twister2_master.bzl", "twister2_master_jersey_files")
@@ -106,6 +107,7 @@ pack_tw2(
         ":twister2-master-lib",
         ":twister2-python-support",
         ":twister2-tset",
+        ":twister2-dl",
     ],
 )
 
@@ -142,6 +144,7 @@ pack_tw2(
         ":twister2-ompi-lib",
         ":twister2-python-support",
         ":twister2-tset",
+        ":twister2-dl",
     ],
 )
 
@@ -339,6 +342,7 @@ pack_tw2(
         ":twister2-python-examples",
         ":twister2-python-support",
         ":twister2-tset",
+        ":twister2-dl",
         ":twister2-ucx-native-libs",
         ":twister2-ucx-native-libs-dir"
     ],
@@ -402,6 +406,7 @@ pack_tw2(
         ":twister2-test-util",
         ":twister2-test-values",
         ":twister2-tset",
+        ":twister2-dl",
         ":twister2-ucx-native-libs",
         ":twister2-ucx-native-libs-dir"
     ],
@@ -638,6 +643,13 @@ pack_tw2(
     version = T2_VERSION,
 )
 
+pack_tw2(
+    name = "twister2-dl",
+    srcs = twister2_core_dl_files(),
+    package_dir = "lib",
+    version = T2_VERSION,
+)
+
 pack_tw2(
     name = "twister2-test",
     srcs = ["//util/test"],

t2_workspace_defs.bzl

@@ -87,6 +87,7 @@ def load_modules():
             "io.netty:netty-all:4.1.27.Final",
             "io.netty:netty-transport:4.1.27.Final",
             "io.netty:netty-transport-native-epoll:4.1.27.Final",
+            "org.apache.commons:commons-math3:3.6.1",
         ],
         repositories = [
             "https://repo1.maven.org/maven2",
@@ -327,3 +328,24 @@ def load_modules():
     _maven_import(artifact = "org.easymock:easymock:4.0.2", licenses = ["notice"], sha256 = "104370107ef64d115e642cbdc14cea438a8d076f5aee9a9cb7882dc9a8ed4123")
     _maven_import(artifact = "org.apache.commons:commons-collections4:4.4", licenses = ["notice"], sha256 = "1df8b9430b5c8ed143d7815e403e33ef5371b2400aadbe9bda0883762e0846d1")
     _maven_import(artifact = "org.apache.commons:commons-configuration2:2.6", licenses = ["notice"], sha256 = "225788911e53af0b29a31a18e0d03b05d86aa9c9e0b3c6686982c30c10f931fb")
+
+
+    #for DL
+    _maven_import(artifact = "org.apache.maven:maven-plugin-api:2.0",licenses = ["notice"],sha256 = "5b62626069d85bb463314572734988d47bc98aab9f0ed48d2f1f9554960f5a35",)
+    _maven_import(artifact = "org.qunix:structure-maven-plugin:0.0.2",licenses = ["notice"],sha256 = "5f07ad70ba250c2fd49937ea6cbc1cbb51d8facd43872fbf65dff47e6d8858fc",)
+    _maven_import(artifact = "org.apache.maven:maven-profile:2.0.8",licenses = ["notice"],sha256 = "bd566c2fdb896e3dd157dc3e49b3c10f93250daaa6462af3cd42ad5b4aeda0a3",)
+    _maven_import(artifact = "org.hamcrest:hamcrest-core:1.3",licenses = ["notice"],sha256 = "66fdef91e9739348df7a096aa384a5685f4e875584cce89386a7a47251c4d8e9",)
+    _maven_import(artifact = "org.apache.maven:maven-plugin-registry:2.0.8",licenses = ["notice"],sha256 = "a7e77626ec4c8382a4c11ba808684402582693df00f0c39f7c0fc02cd1ece9ab",)
+    _maven_import(artifact = "org.apache.maven:maven-repository-metadata:2.0.8",licenses = ["notice"],sha256 = "aff8473e802e4e1c226a777a198f72fbdf7ef36f6f972df6de763b767f652ee1",)
+    _maven_import(artifact = "org.apache.maven:maven-settings:2.0.8",licenses = ["notice"],sha256 = "e1873a36ea2debc0bb6210c3064faae91cf12108e9f8b9845e283022f58ecb46",)
+    _maven_import(artifact = "classworlds:classworlds:1.1",licenses = ["notice"],sha256 = "4e3e0ad158ec60917e0de544c550f31cd65d5a97c3af1c1968bf427e4a9df2e4",)
+    _maven_import(artifact = "org.apache.maven.wagon:wagon-provider-api:1.0",licenses = ["notice"],sha256 = "b28dd1302ac34433d8d1b45fb254e093cd7b47277441af2018c8a3a4d8c1a60d",)
+    _maven_import(artifact = "org.apache.maven:maven-artifact-manager:2.0.8",licenses = ["notice"],sha256 = "c257564b252dc69ff3f3603971164fcb387adbd000818e72c959e6be7b6319cd",)
+    _maven_import(artifact = "org.apache.maven:maven-artifact:2.0.8",licenses = ["notice"],sha256 = "5cf23417cdee6a8e1eb6b9c015c8feea62cedb7dceb3ad098e6869569fcfd1c0",)
+    _maven_import(artifact = "org.apache.maven:maven-model:2.0.8",licenses = ["notice"],sha256 = "51cde4f45d74720eaf567444d88c9ffdfc6896dd58dcc459403f6613d8439255",)
+    _maven_import(artifact = "org.apache.maven:maven-project:2.0.8",licenses = ["notice"],sha256 = "46799ed8812c96f1e651958cbd864155cc8a9ae8d4ebe2392b845d232a910d31",)
+    _maven_import(artifact = "junit:junit:4.11",licenses = ["notice"],sha256 = "90a8e1603eeca48e7e879f3afbc9560715322985f39a274f6f6070b43f9d06fe",)
+    _maven_import(artifact = "org.codehaus.plexus:plexus-utils:1.4.6",licenses = ["notice"],sha256 = "b46fe183dc28526d083287817e6374ad460cef2d780f6096d3ea46caa0a25106",)
+    _maven_import(artifact = "org.apache.commons:commons-lang3:3.3.2",licenses = ["notice"],sha256 = "6b81d10754dadf184d386011486e6509c2cc0c3d33565ced4fb4402b9413d47d",)
+    _maven_import(artifact = "commons-collections:commons-collections:3.2.1",licenses = ["notice"],sha256 = "87363a4c94eaabeefd8b930cb059f66b64c9f7d632862f23de3012da7660047b",)
+    _maven_import(artifact = "com.intel.analytics.bigdl.core.dist:all:0.11.1",licenses = ["notice"],sha256 = "6d5ff2212f362d289cbb3db0b401fe5ad5005a63aafce251252130ce0d0c6a7a",)

tools/rules/twister2_core.bzl

@@ -110,6 +110,12 @@ def twister2_core_tset_files():
         "@maven//:com_google_re2j_re2j"
     ]
 
+def twister2_core_dl_files():
+    return [
+        "//twister2/dl/src/java:twister2dl-java",
+        "@com_intel_analytics_bigdl_core_dist_all//jar",
+    ]
+
 def twister2_storm_files():
     return [
         "//twister2/compatibility/storm:twister2-storm",

twister2/data/src/main/java/edu/iu/dsc/tws/data/api/splits/CSVInputSplit.java

@@ -183,7 +183,7 @@ public void open(Config cfg) throws IOException {
   @Override
   public String nextRecord(Object record) throws IOException {
     if (readLine()) {
-      return readRecord(this.readBuffer, this.currOffset, this.currLen);
+      return readRecord(this.currBuffer, this.currOffset, this.currLen);
     } else {
       this.end = true;
       return null;
@@ -245,7 +245,7 @@ protected final boolean readLine() throws IOException {
         }
         readPos++;
       }
-      if (readPos == limit) {
+      if (readPos == limit && this.splitLength == 0) {
         this.end = true;
       }
       if (delimPos == this.delimiter.length) {

twister2/dl/src/java/BUILD

@@ -0,0 +1,41 @@
+package(default_visibility = ["//visibility:public"])
+
+load("//tools:maven.bzl", "t2_java_lib")
+
+t2_java_lib(
+    name = "twister2dl-java",
+    srcs = glob(["**/*.java"]),
+    artifact_name = "Twiter2 TSet Implementation",
+    deps = [
+        "//twister2/api/src/java/edu/iu/dsc/tws/api/checkpointing:checkpointing-api-java",
+        "//twister2/api/src/java/edu/iu/dsc/tws/api/comms:comms-api-java",
+        "//twister2/api/src/java/edu/iu/dsc/tws/api/compute:task-api-java",
+        "//twister2/api/src/java/edu/iu/dsc/tws/api/config:config-api-java",
+        "//twister2/api/src/java/edu/iu/dsc/tws/api/data:data-api-java",
+        "//twister2/api/src/java/edu/iu/dsc/tws/api/dataset:dataset-api-java",
+        "//twister2/api/src/java/edu/iu/dsc/tws/api/faulttolerance:fault-tolerance-api-java",
+        "//twister2/api/src/java/edu/iu/dsc/tws/api/resource:resource-api-java",
+        "//twister2/api/src/java/edu/iu/dsc/tws/api/tset:tset-api-java",
+        "//twister2/api/src/java/edu/iu/dsc/tws/api/exceptions:exceptions-java",
+        "//twister2/checkpointing/src/java:checkpointing-java",
+        "//twister2/common/src/java:common-java",
+        "//twister2/data/src/main/java:data-java",
+        "//twister2/task/src/main/java:task-java",
+        "//twister2/proto:proto-java",
+        "//twister2/proto/utils:proto-utils-java",
+        "//twister2/master/src/java:master-java",
+        "//twister2/tset/src/java:tset-java",
+        "@com_google_protobuf//:protobuf_java",
+        "@com_google_guava_guava",
+        "@org_apache_hadoop_hadoop_common//jar",
+        "@org_apache_hadoop_hadoop_mapreduce_client_core//jar",
+        "@com_intel_analytics_bigdl_core_dist_all//jar",
+        "@maven//:org_apache_commons_commons_math3",
+        "@maven//:com_google_flatbuffers_flatbuffers_java",
+        "@maven//:javax_annotation_javax_annotation_api",
+        "@maven//:io_netty_netty_buffer",
+        "@maven//:io_netty_netty_all",
+        "@maven//:io_netty_netty_transport",
+        "@maven//:io_netty_netty_transport_native_epoll",
+    ],
+)

twister2/dl/src/java/edu/iu/dsc/tws/dl/criterion/AbstractCriterion.java

@@ -0,0 +1,166 @@
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+package edu.iu.dsc.tws.dl.criterion;
+
+import edu.iu.dsc.tws.dl.data.Activity;
+import edu.iu.dsc.tws.dl.data.tensor.DenseTensor;
+
+/**
+ * [[AbstractCriterion]] is an abstract class the concrete criterion should extend.
+ * `Criterion`s are helpful to train a neural network. Given an input and a target,
+ * they compute the gradient according to a loss function.
+ * <p>
+ * It provides some important method such as `forward`, `backward`, `updateOutput`,
+ * `updateGradInput` frequently used as a criteria. Some of them need to be override
+ * in a concrete criterion class.
+ *
+ * @tparam A represents the input type of the criterion, which an be abstract type [[Activity]],
+ * or concrete type [[Tensor]] or [[Table]]
+ * @tparam B represents the output type of the criterion
+ */
+public abstract class AbstractCriterion<I extends Activity, O extends Activity>
+    implements Criterion {
+  protected I gradInput;
+  protected double output;
+  protected float outputf;
+  protected boolean isFloat;
+  protected SizeAverageStatus sizeAverageStatus = SizeAverageStatus.NONE;
+
+  public AbstractCriterion() {
+    gradInput = (I) new DenseTensor(false);
+    output = 0.0;
+    outputf = 0.0f;
+    isFloat = false;
+  }
+
+  public void toFloat() {
+    isFloat = true;
+    gradInput = (I) new DenseTensor(true);
+  }
+
+  public Activity getGradInput() {
+    return gradInput;
+  }
+
+  public void setGradInput(I gradInput) {
+    this.gradInput = gradInput;
+  }
+
+  public double getOutput() {
+    return output;
+  }
+
+  public float getOutputf() {
+    return outputf;
+  }
+
+  public void setOutput(double output) {
+    this.output = output;
+  }
+
+  public void setOutput(float output) {
+    this.outputf = output;
+  }
+
+  /**
+   * Takes an input object, and computes the corresponding loss of the criterion,
+   * compared with `target`.
+   *
+   * @param input  input data
+   * @param target target
+   * @return the loss of criterion
+   */
+  public double forward(I input, O target) {
+    return updateOutput(input, target);
+  }
+
+  public float forwardf(I input, O target) {
+    return updateOutputf(input, target);
+  }
+
+  /**
+   * Performs a back-propagation step through the criterion, with respect to the given input.
+   *
+   * @param input  input data
+   * @param target target
+   * @return gradient corresponding to input data
+   */
+  public I backward(I input, O target) {
+    return updateGradInput(input, target);
+  }
+
+  /**
+   * Computes the loss using input and objective function. This function
+   * returns the result which is stored in the output field.
+   *
+   * @param input  input of the criterion
+   * @param target target or labels
+   * @return the loss of the criterion
+   */
+  public double updateOutput(I input, O target) {
+    return this.output;
+  }
+
+  public float updateOutputf(I input, O target) {
+    return this.outputf;
+  }
+
+  /**
+   * Computing the gradient of the criterion with respect to its own input. This is returned in
+   * gradInput. Also, the gradInput state variable is updated accordingly.
+   *
+   * @param input  input data
+   * @param target target data / labels
+   * @return gradient of input
+   */
+  public abstract I updateGradInput(I input, O target);
+
+  /**
+   * Deep copy this criterion
+   *
+   * @return a deep copied criterion
+   */
+  AbstractCriterion<I, O> cloneCriterion() {
+    throw new UnsupportedOperationException("Clone not supported");
+  }
+
+  @Override
+  public int hashCode() {
+    //TODO check correctness
+    if (this.isFloat) {
+      return Double.valueOf(outputf).hashCode();
+    } else {
+      return Double.valueOf(output).hashCode();
+    }
+  }
+
+  public boolean canEqual(Object other) {
+    return other instanceof AbstractCriterion;
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (this.isFloat) {
+      return other instanceof AbstractCriterion && other.getClass() == this.getClass()
+          && this.outputf == ((AbstractCriterion<?, ?>) other).getOutput();
+    } else {
+      return other instanceof AbstractCriterion && other.getClass() == this.getClass()
+          && this.output == ((AbstractCriterion<?, ?>) other).getOutput();
+    }
+  }
+
+  enum SizeAverageStatus {
+    TRUE,
+    FALSE,
+    NONE
+  }
+}