From 319720a771d52773d954554a6587321faa2a3d77 Mon Sep 17 00:00:00 2001 From: K Hildrum Date: Fri, 9 May 2014 10:34:24 -0400 Subject: [PATCH 1/4] Attempt at getting maven setup --- .gitignore | 1 + com.ibm.streamsx.hbase/build.xml | 41 ++++++++++++------ com.ibm.streamsx.hbase/impl/java/.gitignore | 2 + .../com/ibm/streamsx/hbase/HBASEOperator.java | 2 +- com.ibm.streamsx.hbase/pom-v094.xml | 42 +++++++++++++++++++ com.ibm.streamsx.hbase/pom-v096-hadoop1.xml | 42 +++++++++++++++++++ com.ibm.streamsx.hbase/pom-v096-hadoop2.xml | 42 +++++++++++++++++++ 7 files changed, 158 insertions(+), 14 deletions(-) create mode 100644 com.ibm.streamsx.hbase/impl/java/.gitignore create mode 100644 com.ibm.streamsx.hbase/pom-v094.xml create mode 100644 com.ibm.streamsx.hbase/pom-v096-hadoop1.xml create mode 100644 com.ibm.streamsx.hbase/pom-v096-hadoop2.xml diff --git a/.gitignore b/.gitignore index b54f413..11ddd5b 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ impl/java/bin/ samples/*/output samples/*/toolkit.xml +/samples/*/doc $StreamsModel.java # begin streamsx.hbase excluded diff --git a/com.ibm.streamsx.hbase/build.xml b/com.ibm.streamsx.hbase/build.xml index 81a4477..91912f3 100644 --- a/com.ibm.streamsx.hbase/build.xml +++ b/com.ibm.streamsx.hbase/build.xml @@ -11,13 +11,16 @@ - - + + + + + @@ -29,6 +32,7 @@ + @@ -40,29 +44,32 @@ + + + + + - + + + + - + + + + - - - - - - - - - + @@ -81,6 +88,14 @@ + + + + + + + + diff --git a/com.ibm.streamsx.hbase/impl/java/.gitignore b/com.ibm.streamsx.hbase/impl/java/.gitignore new file mode 100644 index 0000000..167fb2c --- /dev/null +++ b/com.ibm.streamsx.hbase/impl/java/.gitignore @@ -0,0 +1,2 @@ +/bin +/bin diff --git a/com.ibm.streamsx.hbase/impl/java/src/com/ibm/streamsx/hbase/HBASEOperator.java b/com.ibm.streamsx.hbase/impl/java/src/com/ibm/streamsx/hbase/HBASEOperator.java index 35dad5f..5bb446e 100644 --- a/com.ibm.streamsx.hbase/impl/java/src/com/ibm/streamsx/hbase/HBASEOperator.java +++ b/com.ibm.streamsx.hbase/impl/java/src/com/ibm/streamsx/hbase/HBASEOperator.java @@ -31,7 +31,7 @@ * @author hildrum * */ -@Libraries({"@HBASE_HOME@/lib/*","@HADOOP_HOME@/hadoop-core.jar","@HADOOP_HOME@/lib/*","@HBASE_HOME@/hbase.jar","@HBASE_HOME@/conf"}) +@Libraries({"opt/downloaded/*","@HBASE_HOME@/conf"}) public abstract class HBASEOperator extends AbstractOperator { protected List staticColumnFamilyList= null; protected List staticColumnQualifierList = null; diff --git a/com.ibm.streamsx.hbase/pom-v094.xml b/com.ibm.streamsx.hbase/pom-v094.xml new file mode 100644 index 0000000..f8f09a7 --- /dev/null +++ b/com.ibm.streamsx.hbase/pom-v094.xml @@ -0,0 +1,42 @@ + + + 4.0.0 + com.ibm.streamsx.hbase + streamsx.hbase + jar + 1.2.0 + com.ibm.streamsx.hbase + + + apache.snapshots + http://repository.apache.org/content/repositories/snapshots/ + + + + + + + org.apache.hadoop + hadoop-client + 1.1.1 + + + org.apache.hbase + hbase + 0.94.0 + + + + . + + + + diff --git a/com.ibm.streamsx.hbase/pom-v096-hadoop1.xml b/com.ibm.streamsx.hbase/pom-v096-hadoop1.xml new file mode 100644 index 0000000..e44a3e7 --- /dev/null +++ b/com.ibm.streamsx.hbase/pom-v096-hadoop1.xml @@ -0,0 +1,42 @@ + + + 4.0.0 + com.ibm.streamsx.hbase + streamsx.hbase + jar + 1.2.0 + com.ibm.streamsx.hbase + + + apache.snapshots + http://repository.apache.org/content/repositories/snapshots/ + + + + + + + org.apache.hadoop + hadoop-client + 1.1.1 + + + org.apache.hbase + hbase-client + 0.96.2-hadoop1 + + + + . + + + + diff --git a/com.ibm.streamsx.hbase/pom-v096-hadoop2.xml b/com.ibm.streamsx.hbase/pom-v096-hadoop2.xml new file mode 100644 index 0000000..f129141 --- /dev/null +++ b/com.ibm.streamsx.hbase/pom-v096-hadoop2.xml @@ -0,0 +1,42 @@ + + + 4.0.0 + com.ibm.streamsx.hbase + streamsx.hbase + jar + 1.2.0 + com.ibm.streamsx.hbase + + + apache.snapshots + http://repository.apache.org/content/repositories/snapshots/ + + + + + + + org.apache.hadoop + hadoop-client + 2.2.0 + + + org.apache.hbase + hbase-client + 0.96.2-hadoop2 + + + + . + + + + From 234ff2bcad29ed15e92cc251584149114ce04934 Mon Sep 17 00:00:00 2001 From: K Hildrum Date: Fri, 9 May 2014 10:34:24 -0400 Subject: [PATCH 2/4] Attempt at getting maven setup --- .gitignore | 1 + com.ibm.streamsx.hbase/build.xml | 48 ++++++++++--------- com.ibm.streamsx.hbase/impl/java/.gitignore | 2 + .../com/ibm/streamsx/hbase/HBASEOperator.java | 2 +- com.ibm.streamsx.hbase/pom-v094.xml | 42 ++++++++++++++++ com.ibm.streamsx.hbase/pom-v096-hadoop1.xml | 42 ++++++++++++++++ com.ibm.streamsx.hbase/pom-v096-hadoop2.xml | 42 ++++++++++++++++ 7 files changed, 156 insertions(+), 23 deletions(-) create mode 100644 com.ibm.streamsx.hbase/impl/java/.gitignore create mode 100644 com.ibm.streamsx.hbase/pom-v094.xml create mode 100644 com.ibm.streamsx.hbase/pom-v096-hadoop1.xml create mode 100644 com.ibm.streamsx.hbase/pom-v096-hadoop2.xml diff --git a/.gitignore b/.gitignore index 7490258..0a3fc2b 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ impl/java/bin/ samples/*/output samples/*/toolkit.xml +/samples/*/doc $StreamsModel.java # begin streamsx.hbase excluded diff --git a/com.ibm.streamsx.hbase/build.xml b/com.ibm.streamsx.hbase/build.xml index b3bf2a6..13d01d0 100644 --- a/com.ibm.streamsx.hbase/build.xml +++ b/com.ibm.streamsx.hbase/build.xml @@ -9,15 +9,14 @@ - - - - + + + @@ -29,6 +28,7 @@ + @@ -40,36 +40,32 @@ + + + + + - + + + + - + + + + - - - - - - - - - - - - - - - - + @@ -90,6 +86,14 @@ + + + + + + + + diff --git a/com.ibm.streamsx.hbase/impl/java/.gitignore b/com.ibm.streamsx.hbase/impl/java/.gitignore new file mode 100644 index 0000000..167fb2c --- /dev/null +++ b/com.ibm.streamsx.hbase/impl/java/.gitignore @@ -0,0 +1,2 @@ +/bin +/bin diff --git a/com.ibm.streamsx.hbase/impl/java/src/com/ibm/streamsx/hbase/HBASEOperator.java b/com.ibm.streamsx.hbase/impl/java/src/com/ibm/streamsx/hbase/HBASEOperator.java index abf36e0..4828bcb 100644 --- a/com.ibm.streamsx.hbase/impl/java/src/com/ibm/streamsx/hbase/HBASEOperator.java +++ b/com.ibm.streamsx.hbase/impl/java/src/com/ibm/streamsx/hbase/HBASEOperator.java @@ -38,7 +38,7 @@ * @author hildrum * */ -@Libraries({"opt/local_override/*","@HBASE_HOME@/lib/*","@HADOOP_HOME@/hadoop-core.jar","@HADOOP_HOME@/lib/*","@HBASE_HOME@/hbase.jar","@HBASE_HOME@/conf","opt/local_override/*"}) +@Libraries({"opt/downloaded/*","@HBASE_HOME@/conf"}) public abstract class HBASEOperator extends AbstractOperator { protected List staticColumnFamilyList= null; protected List staticColumnQualifierList = null; diff --git a/com.ibm.streamsx.hbase/pom-v094.xml b/com.ibm.streamsx.hbase/pom-v094.xml new file mode 100644 index 0000000..f8f09a7 --- /dev/null +++ b/com.ibm.streamsx.hbase/pom-v094.xml @@ -0,0 +1,42 @@ + + + 4.0.0 + com.ibm.streamsx.hbase + streamsx.hbase + jar + 1.2.0 + com.ibm.streamsx.hbase + + + apache.snapshots + http://repository.apache.org/content/repositories/snapshots/ + + + + + + + org.apache.hadoop + hadoop-client + 1.1.1 + + + org.apache.hbase + hbase + 0.94.0 + + + + . + + + + diff --git a/com.ibm.streamsx.hbase/pom-v096-hadoop1.xml b/com.ibm.streamsx.hbase/pom-v096-hadoop1.xml new file mode 100644 index 0000000..e44a3e7 --- /dev/null +++ b/com.ibm.streamsx.hbase/pom-v096-hadoop1.xml @@ -0,0 +1,42 @@ + + + 4.0.0 + com.ibm.streamsx.hbase + streamsx.hbase + jar + 1.2.0 + com.ibm.streamsx.hbase + + + apache.snapshots + http://repository.apache.org/content/repositories/snapshots/ + + + + + + + org.apache.hadoop + hadoop-client + 1.1.1 + + + org.apache.hbase + hbase-client + 0.96.2-hadoop1 + + + + . + + + + diff --git a/com.ibm.streamsx.hbase/pom-v096-hadoop2.xml b/com.ibm.streamsx.hbase/pom-v096-hadoop2.xml new file mode 100644 index 0000000..f129141 --- /dev/null +++ b/com.ibm.streamsx.hbase/pom-v096-hadoop2.xml @@ -0,0 +1,42 @@ + + + 4.0.0 + com.ibm.streamsx.hbase + streamsx.hbase + jar + 1.2.0 + com.ibm.streamsx.hbase + + + apache.snapshots + http://repository.apache.org/content/repositories/snapshots/ + + + + + + + org.apache.hadoop + hadoop-client + 2.2.0 + + + org.apache.hbase + hbase-client + 0.96.2-hadoop2 + + + + . + + + + From d9f1dce23390ea4efee8eb4c04a413aa9cbcaf88 Mon Sep 17 00:00:00 2001 From: K Hildrum Date: Tue, 27 May 2014 12:10:54 -0400 Subject: [PATCH 3/4] Merge changes, adjust build to give the correct error messages --- com.ibm.streamsx.hbase/build.xml | 17 +++++++++++++---- com.ibm.streamsx.hbase/pom-v094.xml | 2 +- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/com.ibm.streamsx.hbase/build.xml b/com.ibm.streamsx.hbase/build.xml index 13d01d0..504a262 100644 --- a/com.ibm.streamsx.hbase/build.xml +++ b/com.ibm.streamsx.hbase/build.xml @@ -10,7 +10,8 @@ - + + @@ -29,6 +30,7 @@ + @@ -69,11 +71,9 @@ - - @@ -87,13 +87,22 @@ - + + + + + + + + + + diff --git a/com.ibm.streamsx.hbase/pom-v094.xml b/com.ibm.streamsx.hbase/pom-v094.xml index f8f09a7..80f77de 100644 --- a/com.ibm.streamsx.hbase/pom-v094.xml +++ b/com.ibm.streamsx.hbase/pom-v094.xml @@ -31,7 +31,7 @@ org.apache.hbase hbase - 0.94.0 + 0.94.16 From 877070e25b75b18ba5f2cd43738b718e046f824c Mon Sep 17 00:00:00 2001 From: hildrum Date: Wed, 28 May 2014 11:26:00 -0400 Subject: [PATCH 4/4] maven build instructions Update to reflect build instructions with maven. --- README.md | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 421dc49..b076fc4 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ It uses the operator parameter values to determine what's a row, columnFamily, c It includes at least one sample for each of those operators. -This is tested with HBase 0.94.11 and Hadoop 1.1.0. +This is tested with HBase 0.94.3 and Hadoop 1.1.0, but is expected to work for any later version of hadoop or HBASE. ## Setup @@ -20,14 +20,36 @@ To run these operators, you must install Apache HBASE (https://hbase.apache.org/ The SPLDoc for this toolkit includes some information on getting started once you have HBASE and HADOOP http://ibmstreams.github.io/streamsx.hbase/com.ibm.streamsx.hbase/doc/spldoc/html/index.html -Please see the individual product pages for instructions on installation. +Please see the individual product pages for instructions on installation. + +## Building the toolkit + +The toolkit uses maven (http://maven.apache.org/). Maven will d +* Set M2_HOME to point to the maven directory. +* Pick the correct pom file for your install. In com.ibm.streamsx.hbase, there are three example pom files: + * pom-v094.xml: HBASE 0.94, hadoop 1 + * pom-v096-hadoop1.xml, HBASE 0.96, hadoop 1 + * pom-v096-hadoop2.xml, HBASE 0.96, hadoop 2 + copy the correct file for your HBASE and Hadoop install to pom.xml +* run ant at the top level. This will build the toolkit, but also download all the necessary jars into opt/downloaded. These jars are used at toolkit build time, but also at the toolkit runtime. ##Configuration -To run these operators, you must set `HBASE_HOME` and `HADOOP_HOME` in your environment. Furthermore, `HBASE_HOME/conf/hbase-site.xml` should be correspond to a running hbase instance (eg, the zookeeper referenced there should be running) since that is what the operator uses to configure itself. The operator does not need to run on the same host as HBASE. If you are running Streams on an HBASE host, just set HBASE_HOME to point the the directory for HBASE (ie, the directory that is the parent of the bin directory the conf directory). +To run these operators in an SPL application, you must set `HBASE_HOME`. The operator will look under `HBASE_HOME/conf/hbase-site.xml` for HBASE configuration information, so that file should correspond to a running HBASE instance. The operator does not need to run on the same host as HBASE. If you are running Streams on an HBASE host, just set HBASE_HOME to point the the directory for HBASE (ie, the directory that is the parent of the bin directory the conf directory). If you are running Streams on a different host than where you ran HBASE, copy that whole directory to the Streams host. -The toolkit uses a call first found in HBASE 0.94.11; if your HBASE version is less than than that, download a hbase 0.94.16 jar, and put it in toolkit in com.ibm.streamsx.hbase/opt/local_override. +## Getting started + +One you are started, look under samples at +* PutSample +* PutRecord +* GetSample +* GetRecord + +## Troubleshooting + +If you run into trouble at any point, please enter an issue on GitHub. + ##Contributing This repository is using the fork-and-pull model (https://help.github.com/articles/using-pull-requests). If you'd like to contribute code, fork a copy of the repository, make changes, and when ready, issue a pull request. For more details, see the wiki in the IBMStreams/toolkits repository.