Merge pull request #43 from hildrum/maven

Maven
IBMStreams · May 28, 2014 · 78ee593 · 78ee593
2 parents c5cdece + 877070e
commit 78ee593
Show file tree

Hide file tree

Showing 8 changed files with 189 additions and 29 deletions.
diff --git a/.gitignore b/.gitignore
@@ -15,6 +15,7 @@ impl/java/bin/
 
 samples/*/output
 samples/*/toolkit.xml
+/samples/*/doc
 $StreamsModel.java
 
 # begin streamsx.hbase excluded

diff --git a/README.md b/README.md
@@ -12,22 +12,44 @@ It uses the operator parameter values to determine what's a row, columnFamily, c
 
 It includes at least one sample for each of those operators.
 
-This is tested with HBase 0.94.11 and Hadoop 1.1.0.  
+This is tested with HBase 0.94.3 and Hadoop 1.1.0, but is expected to work for any later version of hadoop or HBASE.  
 
 ## Setup
 
 To run these operators, you must install Apache HBASE (https://hbase.apache.org/).  Apache HBASE usually uses Apache Hadoop (http://hadoop.apache.org/) and Apache Zookeeper (http://zookeeper.apache.org/).  An installation of IBM's BigInsights includes everything you need.  
 
 The SPLDoc for this toolkit includes some information on getting started once you have HBASE and HADOOP http://ibmstreams.github.io/streamsx.hbase/com.ibm.streamsx.hbase/doc/spldoc/html/index.html
 
-Please see the individual product pages for instructions on installation.  
+Please see the individual product pages for instructions on installation. 
+
+## Building the toolkit
+
+The toolkit uses maven (http://maven.apache.org/). Maven will d 
+* Set M2_HOME to point to the maven directory.
+* Pick the correct pom file for your install.  In com.ibm.streamsx.hbase, there are three example pom files: 
+      * pom-v094.xml: HBASE 0.94, hadoop 1
+      * pom-v096-hadoop1.xml, HBASE 0.96, hadoop 1
+      * pom-v096-hadoop2.xml, HBASE 0.96, hadoop 2
+  copy the correct file for your HBASE and Hadoop install to pom.xml
+* run ant at the top level.  This will build the toolkit, but also download all the necessary jars into opt/downloaded.  These jars are used at toolkit build time, but also at the toolkit runtime.
 
 ##Configuration
 
-To run these operators, you must set `HBASE_HOME` and `HADOOP_HOME` in your environment.  Furthermore, `HBASE_HOME/conf/hbase-site.xml` should be correspond to a running hbase instance (eg, the zookeeper referenced there should be running) since that is what the operator uses to configure itself.   The operator does not need to run on the same host as HBASE.  If you are running Streams on an HBASE host, just set HBASE_HOME to point the the directory for HBASE (ie, the directory that is the parent of the bin directory the conf directory).  
+To run these operators in an SPL application, you must set `HBASE_HOME`.  The operator will look under `HBASE_HOME/conf/hbase-site.xml` for HBASE configuration information, so that file should correspond to a running HBASE instance.  The operator does not need to run on the same host as HBASE.  If you are running Streams on an HBASE host, just set HBASE_HOME to point the the directory for HBASE (ie, the directory that is the parent of the bin directory the conf directory).  
 
 If you are running Streams on a different host than where you ran HBASE, copy that whole directory to the Streams host. 
 
-The toolkit uses a call first found in HBASE 0.94.11; if your HBASE version is less than than that, download a hbase 0.94.16 jar, and put it in toolkit in com.ibm.streamsx.hbase/opt/local_override.  
+## Getting started
+
+One you are started, look under samples at  
+* PutSample
+* PutRecord
+* GetSample
+* GetRecord
+
+## Troubleshooting
+
+If you run into trouble at any point, please enter an issue on GitHub.  
+
 ##Contributing
 This repository is using the fork-and-pull model (https://help.github.com/articles/using-pull-requests).  If you'd like to contribute code, fork a copy of the repository, make changes, and when ready, issue a pull request.  For more details, see the wiki in the IBMStreams/toolkits repository.
diff --git a/com.ibm.streamsx.hbase/build.xml b/com.ibm.streamsx.hbase/build.xml
@@ -9,15 +9,14 @@
 <project name="com.ibm.streamsx.hbase" basedir="." default="all">
 
 	<property environment="env" />
-	<!-- you must set HBASE_HOME and HADOOP_HOME in the environment -->
-	<!-- you do not need to have either running base or hdfs running -->
-  <fail unless="env.HADOOP_HOME" message="HADOOP_HOME not set.  This toolkit requres Apache Hadoop, see  http://hadoop.apache.org/.  The toolkit was tested with 1.1.0 and may require modifications for other versions.  If you are running with IBM's BigInsights, sourcing the BIGINSIGHTS_HOME/conf/biginsights-env.sh will set the environment needed."/>
-<!--  <fail unless="env.HBASE_HOME" message="HBASE_HOME not set.  This toolkit requires Apache HBASE, see https://hbase.apache.org/.  This toolkit was tested 0.94.3.  If you are running with IBM's BigInsights, sourcing the BIGINSIGHTS_HOME/conf/biginsights-env.sh will set the environment needed."/> -->
 	<fail unless="env.STREAMS_INSTALL" message="STREAMS_INSTALL not set."/>
+	<available file="${basedir}/pom.xml" property="pomset"/>
 	<property name="streams.install" value="${env.STREAMS_INSTALL}" />
 
 	<property name="hadoop_home" value="${env.HADOOP_HOME}" />
         <property name="hbase_home" value="${env.HBASE_HOME}" />
+	<property name="maven.bin" value="${env.M2_HOME}/bin/mvn" />
+	<property name="ext.downloads.dir" value="opt/downloaded" />
 	<property name="src.dir" value="impl/java/src" />
 	<property name="build.dir" value="impl/java/classes" />
 	<property name="override.dir" value="opt/local_override" />
@@ -31,6 +30,7 @@
 		<mkdir dir="${override.dir}" />
 		<mkdir dir="${lib.dir}" />
 		<mkdir dir="${impl.lib.dir}" />
+		<mkdir dir="${ext.downloads.dir}" />
 	</target>
 
 	<target name="clean">
@@ -42,32 +42,29 @@
 	</target>
 
 
+	<target name="distclean" depends="clean">
+		<delete dir="${ext.downloads.dir}" />
+	</target>
+
+
 	<path id="cp.streams">
 		<pathelement location="${streams.install}/lib/com.ibm.streams.operator.samples.jar" />
 	</path>
 
 	<path id="cp.hbase">
-		<pathelement location="${hbase_home}/hbase.jar" />
+      <fileset dir="${ext.downloads.dir}">
+        <include name="**/hbase*.jar"/>
+      </fileset>
 	</path>
 
+	<!-- Only thing we need here is org.apache.hadoop.conf.Configuration -->
 	<path id="cp.hadoop">
-		<pathelement location="${hadoop_home}/hadoop-core.jar" />
+		<pathelement location="${ext.downloads.dir}/hadoop-core-1.1.1.jar" />
+		<fileset dir="${ext.downloads.dir}">
+		  <include name="hadoop-common*.jar"/>
+		</fileset>
 	</path>
 
-	<path id="cp.local">
-	  <fileset dir ="${basedir}/opt/local_override">
-	    <include name="*.jar"/>
-	    <include name="hbase-site.xml"/>
-	  </fileset>
-	</path>
-
-
-	<path id="cp.compile">
-		<path refid="cp.streams" />
-		<path refid="cp.hbase" />
-		<path refid="cp.hadoop" />
-	</path>
-
 	<target name="all" depends="jar">
 	 <exec executable="${streams.install}/bin/spl-make-toolkit">
            <arg value="--directory"/>
@@ -76,16 +73,13 @@
 	</target>
 
 
-
-	<target name="compile" depends="init">
+	<target name="compile" depends="init,maven-deps">
 		<javac srcdir="${src.dir}" destdir="${build.dir}" debug="true"
 			includeantruntime="no">
 			<classpath>
-			  <path refid="cp.local" />
 				<path refid="cp.streams" />
 				<path refid="cp.hbase" />
 				<path refid="cp.hadoop" />
-				<path refid="cp.local" />
 			</classpath>
 		</javac>
 	</target>
@@ -98,6 +92,21 @@
 		<delete dir="${build.dir}" />
 	</target>
 
+	<!-- Downloads libraries using maven -->
+	<target name="maven-deps" depends="init,nomaven" if="env.M2_HOME">
+		<fail unless="pomset" message="There must be a pom.xml file in com.ibm.streamsx.hbase in order to use maven to download necessary dependencies.  Three sample pom files are included: pom-v094.xml (hbase 0.94, hadoop 1),pom-v096-hadoop1.xml (hbase 0.96, hadoop 1), and  pom-v096-hadoop2.xml (hbase 0.96, hadoop 2).  Please rename the correct one for your install to pom.xml"/> 
+		<exec executable="${maven.bin}"  failonerror="true">
+			<arg value="dependency:copy-dependencies"/>
+			<arg value="-DoutputDirectory=${ext.downloads.dir}"/>
+	    </exec>
+	</target>
+
+
+	<!-- Prints a message if no maven is found. -->
+	<target name="nomaven" unless="env.M2_HOME">
+	  <echo message="M2_HOME not found in environment; expect to find all needed libraries in /opt/downloaded already"/>
+	</target>
+
         <!-- assumes Junit 4.3 (or later) is in $HOME/.ant/lib -->
 	<target name="test" depends="jar">
 	  <!-- empty, for now. -->

diff --git a/com.ibm.streamsx.hbase/impl/java/.gitignore b/com.ibm.streamsx.hbase/impl/java/.gitignore
@@ -0,0 +1,2 @@
+/bin
+/bin
diff --git a/com.ibm.streamsx.hbase/impl/java/src/com/ibm/streamsx/hbase/HBASEOperator.java b/com.ibm.streamsx.hbase/impl/java/src/com/ibm/streamsx/hbase/HBASEOperator.java
@@ -39,7 +39,7 @@
  * @author hildrum
  *
  */
-@Libraries({"opt/local_override/*","@HBASE_HOME@/lib/*","@HADOOP_HOME@/hadoop-core.jar","@HADOOP_HOME@/lib/*","@HBASE_HOME@/hbase.jar","@HBASE_HOME@/conf","opt/local_override/*"})
+@Libraries({"opt/downloaded/*","@HBASE_HOME@/conf"})
 public abstract class HBASEOperator extends AbstractOperator {
 	protected List<String> staticColumnFamilyList= null;
 	protected List<String> staticColumnQualifierList = null;

diff --git a/com.ibm.streamsx.hbase/pom-v094.xml b/com.ibm.streamsx.hbase/pom-v094.xml
@@ -0,0 +1,42 @@
+<!--
+#
+# *******************************************************************************
+# * Copyright (C)2014, International Business Machines Corporation and *
+# * others. All Rights Reserved. *
+# *******************************************************************************
+# 
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.ibm.streamsx.hbase</groupId>
+    <artifactId>streamsx.hbase</artifactId>
+    <packaging>jar</packaging>
+    <version>1.2.0</version>
+    <name>com.ibm.streamsx.hbase</name>
+    <repositories>
+        <repository>
+            <id>apache.snapshots</id>
+            <url>http://repository.apache.org/content/repositories/snapshots/</url>
+        </repository>
+    </repositories>
+    <properties>
+    </properties>
+    <dependencies>
+<dependency>
+	<groupId>org.apache.hadoop</groupId>
+	<artifactId>hadoop-client</artifactId>
+	<version>1.1.1</version>
+</dependency>
+    <dependency>
+	<groupId>org.apache.hbase</groupId>
+	<artifactId>hbase</artifactId>
+	<version>0.94.16</version>
+    </dependency>
+    </dependencies>
+    <build>
+	    <sourceDirectory>.</sourceDirectory>
+        <plugins>
+        </plugins>
+    </build>
+</project>
diff --git a/com.ibm.streamsx.hbase/pom-v096-hadoop1.xml b/com.ibm.streamsx.hbase/pom-v096-hadoop1.xml
@@ -0,0 +1,42 @@
+<!--
+#
+# *******************************************************************************
+# * Copyright (C)2014, International Business Machines Corporation and *
+# * others. All Rights Reserved. *
+# *******************************************************************************
+# 
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.ibm.streamsx.hbase</groupId>
+    <artifactId>streamsx.hbase</artifactId>
+    <packaging>jar</packaging>
+    <version>1.2.0</version>
+    <name>com.ibm.streamsx.hbase</name>
+    <repositories>
+        <repository>
+            <id>apache.snapshots</id>
+            <url>http://repository.apache.org/content/repositories/snapshots/</url>
+        </repository>
+    </repositories>
+    <properties>
+    </properties>
+    <dependencies>
+<dependency>
+	<groupId>org.apache.hadoop</groupId>
+	<artifactId>hadoop-client</artifactId>
+	<version>1.1.1</version>
+</dependency>
+<dependency>
+  <groupId>org.apache.hbase</groupId>
+  <artifactId>hbase-client</artifactId>
+  <version>0.96.2-hadoop1</version>
+</dependency>
+    </dependencies>
+    <build>
+	    <sourceDirectory>.</sourceDirectory>
+        <plugins>
+        </plugins>
+    </build>
+</project>
diff --git a/com.ibm.streamsx.hbase/pom-v096-hadoop2.xml b/com.ibm.streamsx.hbase/pom-v096-hadoop2.xml
@@ -0,0 +1,42 @@
+<!--
+#
+# *******************************************************************************
+# * Copyright (C)2014, International Business Machines Corporation and *
+# * others. All Rights Reserved. *
+# *******************************************************************************
+# 
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.ibm.streamsx.hbase</groupId>
+    <artifactId>streamsx.hbase</artifactId>
+    <packaging>jar</packaging>
+    <version>1.2.0</version>
+    <name>com.ibm.streamsx.hbase</name>
+    <repositories>
+        <repository>
+            <id>apache.snapshots</id>
+            <url>http://repository.apache.org/content/repositories/snapshots/</url>
+        </repository>
+    </repositories>
+    <properties>
+    </properties>
+    <dependencies>
+<dependency>
+	<groupId>org.apache.hadoop</groupId>
+	<artifactId>hadoop-client</artifactId>
+	<version>2.2.0</version>
+</dependency>
+<dependency>
+  <groupId>org.apache.hbase</groupId>
+  <artifactId>hbase-client</artifactId>
+  <version>0.96.2-hadoop2</version>
+</dependency>
+    </dependencies>
+    <build>
+	    <sourceDirectory>.</sourceDirectory>
+        <plugins>
+        </plugins>
+    </build>
+</project>