diff --git a/.gitignore b/.gitignore index 697bb46..f3abaff 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,6 @@ vf.gf.* *.log **/.idea/**/*.* **/.gradle/**/*.* -**/gradle/**/*.* **/.class **/.Rapp.history **/.settings @@ -19,3 +18,5 @@ vf.gf.* **/locator/**/*.* start-scdf-*.sh dump.rdb +spring-cloud-dataflow-server-cloudfoundry-1.0.0.RELEASE.jar +spring-cloud-dataflow-shell-1.0.0.RELEASE.jar diff --git a/ClusteringService/build.gradle b/ClusteringService/build.gradle index 6a01ae8..3c1d1d1 100644 --- a/ClusteringService/build.gradle +++ b/ClusteringService/build.gradle @@ -1,6 +1,6 @@ buildscript { ext { - springBootVersion = '1.3.3.RELEASE' + springBootVersion = '1.3.7.RELEASE' } repositories { mavenCentral() @@ -18,6 +18,7 @@ apply plugin: 'eclipse' apply plugin: 'java' apply plugin: 'spring-boot' apply plugin: "io.spring.dependency-management" +apply plugin: "maven" sourceCompatibility = 1.8 @@ -58,15 +59,15 @@ dependencies { compile "org.scala-lang:scala-library:$scalaVersion" compile ('org.apache.spark:spark-core_2.10:1.6.0'){ - exclude group: 'com.fasterxml.jackson.core' - exclude group: 'com.fasterxml.jackson.module' - exclude group: 'org.slf4j' - exclude group: 'org.eclipse.jetty.orbit' - } + exclude group: 'com.fasterxml.jackson.core' + exclude group: 'com.fasterxml.jackson.module' + exclude group: 'org.slf4j' + exclude group: 'org.eclipse.jetty.orbit' + } compile ('org.apache.spark:spark-mllib_2.10:1.6.0'){ - exclude group: 'com.fasterxml.jackson.core' + exclude group: 'com.fasterxml.jackson.core' exclude group: 'com.fasterxml.jackson.module' exclude group: 'org.slf4j' exclude group: 'org.eclipse.jetty.orbit' @@ -74,8 +75,7 @@ dependencies { compile('org.springframework.boot:spring-boot-starter-web'){ exclude group: 'com.fasterxml.jackson.core' - exclude (group: 'org.springframework.boot', module: 'spring-boot-starter-logging') - + exclude (group: 'org.springframework.boot', module: 'spring-boot-starter-logging') } compile ('com.fasterxml.jackson.core:jackson-databind:2.6.5') @@ -94,8 +94,9 @@ dependencies { exclude group: 'com.fasterxml.jackson.core' } - compile ('com.redislabs:spark-redis:0.1.1'){ - exclude group: 'org.slf4j' +// compile ('com.redislabs:spark-redis:0.1.1'){ + compile ('RedisLabs:spark-redis:0.3.2'){ + exclude group: 'org.slf4j' exclude group: 'org.scala-lang' exclude group: 'org.apache.spark' } @@ -119,6 +120,10 @@ tasks.withType(ScalaCompile) { scalaCompileOptions.useAnt = false } +repositories { + maven { + url "http://dl.bintray.com/spark-packages/maven" + } +} - - +install.dependsOn(bootRepackage) diff --git a/ClusteringService/gradle/wrapper/gradle-wrapper.jar b/ClusteringService/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000..30d399d Binary files /dev/null and b/ClusteringService/gradle/wrapper/gradle-wrapper.jar differ diff --git a/ClusteringService/gradle/wrapper/gradle-wrapper.properties b/ClusteringService/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..481bdf6 --- /dev/null +++ b/ClusteringService/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Tue Sep 06 17:18:11 CEST 2016 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-2.5-bin.zip diff --git a/Enrich-processor/build.gradle b/Enrich-processor/build.gradle index 8047344..ccb82a0 100644 --- a/Enrich-processor/build.gradle +++ b/Enrich-processor/build.gradle @@ -51,7 +51,4 @@ eclipse { } } - - - - +install.dependsOn(bootRepackage) \ No newline at end of file diff --git a/Enrich-processor/gradle/wrapper/gradle-wrapper.jar b/Enrich-processor/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000..30d399d Binary files /dev/null and b/Enrich-processor/gradle/wrapper/gradle-wrapper.jar differ diff --git a/Enrich-processor/gradle/wrapper/gradle-wrapper.properties b/Enrich-processor/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..65d9746 --- /dev/null +++ b/Enrich-processor/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Wed Sep 07 18:08:27 CEST 2016 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-2.4-bin.zip diff --git a/GemfireServer/build.gradle b/GemfireServer/build.gradle index 5616979..ea2b865 100644 --- a/GemfireServer/build.gradle +++ b/GemfireServer/build.gradle @@ -10,7 +10,7 @@ buildscript { apply plugin: 'spring-boot' apply plugin: 'java' apply plugin: 'eclipse' - +// apply plugin: "maven" repositories { mavenCentral() @@ -19,13 +19,10 @@ repositories { maven { url "http://repo.spring.io/libs-snapshot" } } - dependencies { - compile ("io.pivotal.gemfire:gemfire-greenplum:1.0.0-beta-6-SNAPSHOT") compile ("com.gemstone.gemfire:gemfire:8.1.0") - compile ("org.apache.logging.log4j:log4j-core:2.4") - + compile ("org.apache.logging.log4j:log4j-core:2.4") } jar { @@ -41,7 +38,6 @@ task serverJar(type: Jar, dependsOn: jar){ bootRepackage.withJarTask = jar - artifacts{ archives jar, serverJar } @@ -49,3 +45,5 @@ artifacts{ task wrapper(type: Wrapper) { gradleVersion = '2.3' } + +//install.dependsOn(bootRepackage) diff --git a/GemfireServer/gradle/wrapper/gradle-wrapper.jar b/GemfireServer/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000..30d399d Binary files /dev/null and b/GemfireServer/gradle/wrapper/gradle-wrapper.jar differ diff --git a/GemfireServer/gradle/wrapper/gradle-wrapper.properties b/GemfireServer/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..65d9746 --- /dev/null +++ b/GemfireServer/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Wed Sep 07 18:08:27 CEST 2016 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-2.4-bin.zip diff --git a/GemfireServer/startup.gfsh b/GemfireServer/startup.gfsh index e8d63d6..425b90c 100644 --- a/GemfireServer/startup.gfsh +++ b/GemfireServer/startup.gfsh @@ -1,2 +1,2 @@ -start locator --name=locator --J=-Dgemfire.http-service-port=7575 --hostname-for-clients=192.168.11.1 -start server --name=server1 --cache-xml-file=src/main/resources/server-cache.xml --J=-Dgemfire.start-dev-rest-api=true --J=-Dgemfire.http-service-bind-address=192.168.11.1 --J=-Dgemfire.http-service-port=8888 --locators=192.168.11.1[10334] --hostname-for-clients=192.168.11.1 --server-bind-address=192.168.11.1 +start locator --name=locator --hostname-for-clients=192.168.11.1 --J=-Dgemfire.http-service-port=7575 +start server --name=server1 --cache-xml-file=src/main/resources/server-cache.xml --locators=192.168.0.10[10334] --hostname-for-clients=192.168.11.1 --server-bind-address=192.168.11.1 --J=-Dgemfire.start-dev-rest-api=true --J=-Dgemfire.http-service-bind-address=192.168.11.1 --J=-Dgemfire.http-service-port=8888 diff --git a/PR-115.patch b/PR-115.patch deleted file mode 100644 index fb78cea..0000000 --- a/PR-115.patch +++ /dev/null @@ -1,37 +0,0 @@ -From 12016187e934aa83e7bdf959c1269b3488e9b1d0 Mon Sep 17 00:00:00 2001 -From: Frederico Melo -Date: Wed, 1 Jun 2016 23:53:54 -0500 -Subject: [PATCH] A fix for issue - https://github.com/spring-cloud/spring-cloud-stream-app-starters/issues/8 - ---- - .../stream/app/pmml/processor/PmmlProcessorConfiguration.java | 9 ++++++++- - 1 file changed, 8 insertions(+), 1 deletion(-) - -diff --git a/processor/spring-cloud-starter-stream-processor-pmml/src/main/java/org/springframework/cloud/stream/app/pmml/processor/PmmlProcessorConfiguration.java b/processor/spring-cloud-starter-stream-processor-pmml/src/main/java/org/springframework/cloud/stream/app/pmml/processor/PmmlProcessorConfiguration.java -index e3b9384..c361a92 100644 ---- a/processor/spring-cloud-starter-stream-processor-pmml/src/main/java/org/springframework/cloud/stream/app/pmml/processor/PmmlProcessorConfiguration.java -+++ b/processor/spring-cloud-starter-stream-processor-pmml/src/main/java/org/springframework/cloud/stream/app/pmml/processor/PmmlProcessorConfiguration.java -@@ -71,6 +71,7 @@ - public class PmmlProcessorConfiguration { - - private static final Log logger = LogFactory.getLog(PmmlProcessorConfiguration.class); -+ private static final String DEFAULT_OUTPUT_FIELD = "_output"; - - private final ModelEvaluatorFactory modelEvaluatorFactory = ModelEvaluatorFactory.newInstance(); - -@@ -122,7 +123,13 @@ public Object evaluate(Message input) { - MutableMessage result = convertToMutable(input); - - for (Map.Entry entry : results.entrySet()) { -- String fieldName = entry.getKey().getValue(); -+ -+ String fieldName = null; -+ if (entry.getKey()==null) -+ fieldName = DEFAULT_OUTPUT_FIELD; -+ else -+ fieldName = entry.getKey().getValue(); -+ - Expression expression = properties.getOutputs().get(fieldName); - if (expression == null) { - expression = spelExpressionParser.parseExpression("payload." + fieldName); diff --git a/README.adoc b/README.adoc index 9066c06..8c7ca6b 100644 --- a/README.adoc +++ b/README.adoc @@ -7,128 +7,162 @@ image::arch-2.jpg[Architecture] === Requirements - - CloudFoundry (tested with pcfdev v0.66) - - Apache Geode (tested with v1.0.0-incubating.M2) - to be replaced by GemFire 9.0 when that is released - - PostgreSQL (tested with 9.5.3) - to be replaced by Greenplum server once GemFire 9.0 is released - - Spring Cloud Dataflow (tested with 1.0.0.M3) - - Spark-redis (tested with v0.1.1) - installed in local maven repository +* CloudFoundry (tested with pcfdev v0.66) +* Apache Geode (tested with http://apache.org/dyn/closer.cgi/incubator/geode/1.0.0-incubating.M3/apache-geode-1.0.0-incubating.M3.tar.gz[apache-geode-1.0.0-incubating.M3] ) - to be replaced by GemFire 9.0 when that is released +* PostgreSQL (tested with 9.5.3) - to be replaced by Greenplum server once GemFire 9.0 is released +* Spring Cloud Dataflow (tested with 1.0.0.RELASE) +* Spark-redis (tested with v0.1.1) - installed from binray maven repository === Preparing the environment - -* Export the variable __$GEODE_HOME__, pointing to your local GemFire / Geode server home. -* Edit the file __start-scdf-pcfdev.sh__, in order to satisfy your pcfdev environment info and SCDF binaries path -* Edit the file __GemFireServer/startup.gfsh__, replacing the IPs with your local address on the pcfdev network (default is 192.168.11.1, for the pcfdev server running on 192.168.11.11). If in doubt, check your pcfdev VM Ip address. -* Edit the file __TransactionsEmulator/src/main/resources/application.properties__, replacing the __geodeUrl__ variable with the correct Geode IP. -* Start pcfdev, if not running yet. -* Start Greenplum (or PostgreSQL) -* Use the DDL statements in __scripts/model_postgres.sql__ to create the database model. -* Import the data from tables using the csv files __scripts/pos_device.csv__, __scripts/zip_codes_states.csv__ and __scripts/transaction.csv__. This step won't be necessary when GemFire 9.0 is released (it will write transactions directly to Greenplum). -* Start GemFire/Geode using the script __GemFireServer/startup.sh__ -* Start the Spring Cloud Dataflow server (or deploy it to pcfdev, if you prefer) - -=== Additional temporary step: patching the PMML model - -There's a known bug in SCDF's PMML app for which I've created the following PR: https://github.com/spring-cloud/spring-cloud-stream-app-starters/pull/115 - -That was already merged into the head, but the latest version isn't stable yet. So for now we'll patch the M1 version with our fix. -This will also make sure all apps are registered locally, so you depend less on good internet connectivity for the demo. +* Download and install http://apache.org/dyn/closer.cgi/incubator/geode/1.0.0-incubating.M3/apache-geode-1.0.0-incubating.M3.tar.gz[Apache Geode M3]. Export the variable `$GEODE_HOME`, pointing to your local `GemFire`/`Geode` server home. +``` +export GEODE_HOME= +export PATH=$GEODE_HOME/bin:$PATH +``` + +* Edit the file `start-scdf-pcfdev.sh`, in order to satisfy your pcfdev environment info and SCDF binaries path. +** Download the SCDF server from: http://repo.spring.io/libs-release/org/springframework/cloud/spring-cloud-dataflow-server-cloudfoundry/1.0.0.RELEASE/spring-cloud-dataflow-server-cloudfoundry-1.0.0.RELEASE.jar[spring-cloud-dataflow-server-cloudfoundry-1.0.0.RELEASE.jar]. +** Download the SCDF shell from: http://repo.spring.io/libs-release/org/springframework/cloud/spring-cloud-dataflow-shell/1.0.0.RELEASE/spring-cloud-dataflow-shell-1.0.0.RELEASE.jar[spring-cloud-dataflow-shell-1.0.0.RELEASE.jar] +* Edit the file `GemFireServer/startup.gfsh`, replacing the IPs with your local address on the pcfdev network (default is 192.168.11.1, for the pcfdev server running on 192.168.11.11). If in doubt, check your pcfdev VM Ip address. +* Edit the file `TransactionsEmulator/src/main/resources/application.properties`, replacing the `geodeUrl` variable with the correct Geode IP. +* Start `pcfdev`, if not running yet. At least 6GB of memory is required! +``` +cf dev start -m 6144 +cf login -a https://api.local.pcfdev.io --skip-ssl-validation +``` +* Start `Greenplum` or `PostgreSQL`. See <> for instructions how to configure PostgreSQL. +** Use the DDL statements in `scripts/model_postgres.sql` to create the database model. +** Import the data from tables using the csv files `scripts/pos_device.csv`, `scripts/zip_codes_states.csv` and `scripts/transaction.csv`. This step won't be necessary when GemFire 9.0 is released (it will write transactions directly to Greenplum). +* Start GemFire/Geode using the script `GemFireServer/startup.sh` +* Start the Spring Cloud Dataflow server (or deploy it to pcfdev, if you prefer). Use the `start-scdf-pcdev.sh` script. Open http://192.168.11.1:9393/dashboard/index.html to acces the SCDF dashboard. + +=== Buils all SDDF 1.0.2 apps localy. Enable the PMML processor. * Clone the repository https://github.com/spring-cloud/spring-cloud-stream-app-starters -* Cd into the repository and apply the patch, by invoking ----- -Frederico-Melos-MacBook-Pro:spring-cloud-stream-app-starters fmelo$ git am ~/FraudDetection-DataMicroservices/PR-115.patch ----- -Replace the patch file path with your local information. - -* Build and install the repository (__./mvnw -s .settings.xml -U clean install -DskipTests__) -* Generate the apps (run ./generateApps.sh) -* Cd into the __apps__ directory. -* Build and install the apps (__./mvnw -s ../.settings.xml -U clean install -DskipTests__) - +``` +git clone https://github.com/spring-cloud/spring-cloud-stream-app-starters +cd spring-cloud-stream-app-starters +git fetch +git checkout tags/v1.0.2.RELEASE +``` + +* In `spring-cloud-stream-app-generator/pom.xml` uncomment the `PMML` processor: +``` + + org.springframework.cloud.stream.app.test.pmml.PmmlProcessorTestConfiguration.class + +``` + +* Build all SCDF apps and install them in the local maven repository +``` +./mvnw clean install -DskipTests +./generateApps.sh +./mvnw clean install -f ./apps/pom.xml -DskipTests +``` === Running the app ==== Setup -1- Create the gemfire user-provided service in CF using the script __TransactionsConsole/cf-createservice.txt__ . If necessary, modify it to point to your correct GemFire server - -2- Create the gpdb user-provided service in CF using the script __ClusteringService/cf-createservice.txt__ . If necessary, modify it to point to your correct Greenplum/PostgreSQL server - -3- Create a RabbitMQ service on CF called "rabbit". This will be our SCDF transport. ----- -Frederico-Melos-MacBook-Pro:FraudDetection-DataMicroservices fmelo$ cf create-service p-rabbitmq standard rabbit -Creating service instance rabbit in org pcfdev-org / space pcfdev-space as admin... -OK ----- -4- Create a Redis service on CF called "redis". This will be used by both the ML Clustering app and the custom enricher - ----- -Frederico-Melos-MacBook-Pro:FraudDetection-DataMicroservices fmelo$ cf create-service p-redis shared-vm redis -Creating service instance redis in org pcfdev-org / space pcfdev-space as admin... -OK ----- - -5- Push the app TransactionsConsole to CF. There's a __manifest.yml__ file on the application root folder. - -6- Open the TransactionsConsole UI (see the CF assigned host, probably http://transactions-console.local.pcfdev.io) and notice the US Map. - -7- Now start the TransactionsEmulator app. You can run __"./gradlew bootRun"__ to start it. Wait for it to be done with loading the devices and transactions start to be posted. - -8- Switch back to the TransactionsConsole UI and see transactions happening. - +1 - Create the `gemfire` user-provided service in CF using the script `TransactionsConsole/cf-createservice.txt`. If necessary, modify it to point to your correct GemFire server. +``` +cf cups gemfire -p '{"locatorHost":"192.168.11.1","locatorPort":"10334", "RestEndpoint":"http://192.168.11.1:8888/gemfire-api/v1/"}' +``` + +2 - Create the `gpdb` user-provided service in CF using the script `ClusteringService/cf-createservice.txt`. If necessary, modify the Greenplum/PostgreSQL server address and access credentials. +``` +cf cups gpdb -p '{"URL":"jdbc:postgresql://192.168.11.1:5432/gemfire?user=pivotal&password=pivotal"}' +``` + +3 - Create a RabbitMQ service on CF called `rabbit`. This will be our SCDF transport. +``` +cf create-service p-rabbitmq standard rabbit +``` + +4 - Create a Redis service on CF called `redis`. This will be used by both the ML Clustering app and the custom enricher. +``` +cf create-service p-redis shared-vm redis +``` + +5 - Build and push the app `TransactionsConsole` to CF. There's a `manifest.yml` file on the application root folder. +``` +cd ./TransactionsConsole +./gradlew clean build -x test +cf push -f ./manifest.yml +cd .. +``` + +6 - Open the `TransactionsConsole UI` (see the CF assigned host, probably http://transactions-console.local.pcfdev.io) and notice the US Map. + +7 - Now build and start the `TransactionsEmulator` app. Wait for it to be done with loading the devices and transactions start to be posted. +``` +cd TransactionsEmulator +./gradlew clean build -x test +java -jar ./build/libs/PoS_Emulator-1.0.0.BUILD-SNAPSHOT.jar +``` + +8 - Switch back to the `TransactionsConsole UI` and see transactions happening. ==== Creating and training the Machine Learning model -9- Push the application ClusteringService to CF. There's a __manifest.yml__ file on the application root folder. +9 - Push the application `ClusteringService` to CF. +``` +cd ./ClusteringService +./gradlew clean build -x test +cf push -f ./manifest.yml +cd .. +``` -10- Once the app is running, trigger the model training by accessing the __/clustering/train__ URL (probably http://clustering-service.local.pcfdev.io/clustering/train). Check the CF logs for that app and see spark querying Greenplum/PostgreSQL and building the model. It will take a few minutes until the model is trained and the URL will return a few numbers. +10 - Once the app is running, trigger the model training by accessing the `/clustering/train` URL (probably http://clustering-service.local.pcfdev.io/clustering/train). Check the CF logs for that app and see spark querying Greenplum/PostgreSQL and building the model. It will take a few minutes until the model is trained and the URL will return a few numbers. -11- Now the model is trained, check its PMML representation using the url __/clustering/model.pmml.xml__ (http://clustering-service.local.pcfdev.io/clustering/model.pmml.xml) +11 - Now the model is trained, check its PMML representation using the url `/clustering/model.pmml.xml` (http://clustering-service.local.pcfdev.io/clustering/model.pmml.xml) -12- Notice the clusters assigned on the three ____ tags by the bottom of the XML. Each cluster is a combination of average values for __"distance from home location"__ and __"transaction value"__ variables. The cluster with the lowest values is likely the one assigned to __low risk transactions__. +12 - Notice the clusters assigned on the three `` tags by the bottom of the XML. Each cluster is a combination of average values for `distance from home location` and `transaction value` variables. The cluster with the lowest values is likely the one assigned to `low risk transactions`. When evaluating the PMML model, each transaction will be assigned to the closest cluster, based on its distance from the home location (where customer does most transactions) and transaction value. ==== Creating the first two streams in SCDF +Start SCDF shell: +``` +java -Xmx128M -jar ./spring-cloud-dataflow-shell-1.0.0.RELEASE.jar +``` -13- Using the SCDF shell, import the common applications by running - ----- -dataflow:>app import --uri file:////Users/fmelo/FraudDetection-DataMicroservices/scripts/scdf-stream-apps.properties - -Successfully registered applications: [source.tcp, sink.jdbc, source.http, sink.rabbit, source.rabbit, source.ftp, sink.gpfdist, processor.transform, source.sftp, processor.filter, source.file, sink.cassandra, processor.groovy-filter, sink.router, source.trigger, sink.hdfs-dataset, processor.splitter, source.load-generator, sink.file, source.time, source.gemfire, source.twitterstream, sink.tcp, source.jdbc, sink.field-value-counter, sink.redis-pubsub, sink.hdfs, processor.bridge, processor.pmml, processor.httpclient, source.s3, sink.ftp, sink.log, sink.gemfire, sink.aggregate-counter, sink.throughput, source.triggertask, source.gemfire-cq, source.jms, processor.scriptable-transform, sink.counter, sink.websocket, source.mail, processor.groovy-transform, source.syslog] ----- +13 - Using the SCDF shell, import the common applications by running +``` +dataflow:>app import --uri http://bit.ly/1-0-2-GA-stream-applications-rabbit-maven +``` -Make sure you replace the path in the command with your correct file location. +14 - Build and install the `Enricher-processor` application and register it with SCDF. +``` +cd Enricher-processor +./gradlew build install -x test +``` -14- Build and install (./gradlew build install) the Enricher-processor application and register it with SCDF. - ----- +Register the Enricher in SCDF: +``` dataflow:>app register --name enrich --type processor --uri maven://io.pivotal.demo:enricher-processor:1.0.0.BUILD-SNAPSHOT --force -Successfully registered application 'processor:enrich' ----- +``` -15- Create the first streams on either the SCDF Shell or the SCDF UI. On the UI, use the following DSL: +15 - Create the first streams on either the SCDF Shell or the SCDF UI. On the UI, use the following DSL: ----- +``` fromgem = gemfire --region-name=Transaction --host-addresses=geode-server:10334 | enrich | log eval = :fromgem.enrich > pmml --modelLocation=http://clustering-service.local.pcfdev.io/clustering/model.pmml.xml --inputs='field_0=payload.distance.doubleValue(),field_1=payload.value.doubleValue()' --inputType='application/x-spring-tuple' --outputType='application/json' | log +``` ----- - -Replace the host-addresses parameter with your GemFire locator information on the first flow. +Replace the `host-addresses` parameter with your GemFire locator address (usually `192.168.11.1`) Notice in the second flow we're mapping the PMML model inputs in the PMML XML file with the properties in our enriched payload. Make sure the modelLocation variable has the right URL to your PMML model endpoint. ==== Deploying the first stream - enriching the payload -16- Deploy the first flow and see now you have three new applications running on CF +16 - Deploy the first flow and see now you have three new applications running on CF ----- -Frederico-Melos-MacBook-Pro:FraudDetection-DataMicroservices fmelo$ cf apps +``` +$ cf apps Getting apps in org pcfdev-org / space pcfdev-space as admin... OK @@ -138,29 +172,28 @@ clustering-service started 1/1 1G 512M cluster dataflow-fromgem-log started 1/1 512M 512M dataflow-fromgem-log.local.pcfdev.io dataflow-fromgem-enrich started 1/1 512M 512M dataflow-fromgem-enrich.local.pcfdev.io dataflow-fromgem-gemfire started 1/1 512M 512M dataflow-fromgem-gemfire.local.pcfdev.io ----- +``` -17- Once the apps are running, start tailing the logs of the "dataflow-fromgem-log" app and then run the TransactionsEmulator again (see step 7 above). You should start seeing transactions + enriched data being logged. +17 - Once the apps are running, start tailing the logs of the "dataflow-fromgem-log" app and then run the TransactionsEmulator again (see step 7 above). You should start seeing transactions + enriched data being logged. ----- -Frederico-Melos-MacBook-Pro:FraudDetection-DataMicroservices fmelo$ cf logs dataflow-fromgem-log +``` +$ cf logs dataflow-fromgem-log Connected, tailing logs for app dataflow-fromgem-log in org pcfdev-org / space pcfdev-space as admin... 2016-06-22T17:28:50.52-0700 [APP/0] OUT 2016-06-23 00:28:50.515 INFO 14 --- [nrich.fromgem-1] log.sink : {"id":8662602513688694487,"deviceId":10,"value":-1.345934346296312E64,"accountId":-61,"timestamp":1008806322260060363,"homeLocation":null,"homeLatitude":0.0,"homeLongitude":0.0,"distance":0.0} 2016-06-22T17:28:50.61-0700 [APP/0] OUT 2016-06-23 00:28:50.611 INFO 14 --- [nrich.fromgem-1] log.sink : {"id":5199577666956545635,"deviceId":12,"value":1.599846306874403E-148,"accountId":-102,"timestamp":432345569956636875,"homeLocation":null,"homeLatitude":0.0,"homeLongitude":0.0,"distance":0.0} 2016-06-22T17:28:50.66-0700 [APP/0] OUT 2016-06-23 00:28:50.657 INFO 14 --- [nrich.fromgem-1] log.sink : {"id":7175376153652552608,"deviceId":6,"value":1.8057620689412976E218,"accountId":51,"timestamp":504403163994564811,"homeLocation":"32.373788:-86.289182","homeLatitude":32.373788,"homeLongitude":-86.289182,"distance":63.24} 2016-06-22T17:28:50.71-0700 [APP/0] OUT 2016-06-23 00:28:50.708 INFO 14 --- [nrich.fromgem-1] log.sink : {"id":7442228185422431077,"deviceId":2,"value":3.567804161380589E-105,"accountId":82,"timestamp":1945555044753123532,"homeLocation":"39.787529:-98.20595","homeLatitude":39.787529,"homeLongitude":-98.20595,"distance":1429.1} +``` ----- - -Notice the "homeLocation" and "distance" attributes on the payload. They were added by the enricher processor. +Notice the `homeLocation` and `distance` attributes on the payload. They were added by the enricher processor. ==== Deploying the second stream to SCDF - evaluating against the PMML model -18- Deploy the second stream (__eval__) and see now you have two new applications running on CF +18 - Deploy the second stream `eval` and see now you have two new applications running on CF ----- -Frederico-Melos-MacBook-Pro:FraudDetection-DataMicroservices fmelo$ cf apps +``` +$ cf apps Getting apps in org pcfdev-org / space pcfdev-space as admin... OK @@ -172,34 +205,34 @@ dataflow-fromgem-enrich started 1/1 512M 512M dataflo dataflow-fromgem-gemfire started 1/1 512M 512M dataflow-fromgem-gemfire.local.pcfdev.io dataflow-eval-log started 1/1 512M 512M dataflow-eval-log.local.pcfdev.io dataflow-eval-pmml started 1/1 512M 512M dataflow-eval-pmml.local.pcfdev.io ----- +``` -19- Once the apps are running, start tailing the logs of the "dataflow-eval-log" app and then run the TransactionsEmulator again (see step 7 above). You should now see the first results of the PMML model evaluation +19 - Once the apps are running, start tailing the logs of the `dataflow-eval-log` app and then run the `TransactionsEmulator` again (see step 7 above). You should now see the first results of the PMML model evaluation. ----- -Frederico-Melos-MacBook-Pro:FraudDetection-DataMicroservices fmelo$ cf logs dataflow-eval-log +``` +$ cf logs dataflow-eval-log Connected, dumping recent logs for app dataflow-eval-log in org pcfdev-org / space pcfdev-space as admin... 2016-06-22T17:28:50.58-0700 [APP/0] OUT 2016-06-23 00:28:50.582 INFO 15 --- [val.pmml.eval-1] log.sink : {"id":8662602513688694487,"deviceId":10,"value":-1.345934346296312E64,"accountId":-61,"timestamp":1008806322260060363,"homeLocation":null,"homeLatitude":0.0,"homeLongitude":0.0,"distance":0.0,"_output":{"result":"1","type":"DISTANCE","entityRegistry":{"1":{"locator":null,"id":null,"name":"cluster_0","size":null,"extensions":[],"kohonenMap":null,"array":{"locator":null,"n":2,"type":"REAL","value":"27.685449231686356 3.349155420943788"},"partition":null,"covariances":null},"2":{"locator":null,"id":null,"name":"cluster_1","size":null,"extensions":[],"kohonenMap":null,"array":{"locator":null,"n":2,"type":"REAL","value":"32.16548772032409 2.544008885888239"},"partition":null,"covariances":null},"3":{"locator":null,"id":null,"name":"cluster_2","size":null,"extensions":[],"kohonenMap":null,"array":{"locator":null,"n":2,"type":"REAL","value":"27.691067777235084 1.7439567824479112"},"partition":null,"covariances":null}},"entity":{"locator":null,"id":null,"name":"cluster_0","size":null,"extensions":[],"kohonenMap":null,"array":{"locator":null,"n":2,"type":"REAL","value":"27.685449231686356 3.349155420943788"},"partition":null,"covariances":null},"categoryValues":["1","2","3"],"entityIdRanking":["1","2","3"],"affinityRanking":[1.811539264540081E128,1.811539264540081E128,1.811539264540081E128],"entityAffinity":1.811539264540081E128,"displayValue":"cluster_0","entityId":"1"}} 2016-06-22T17:28:50.59-0700 [APP/0] OUT 2016-06-23 00:28:50.591 INFO 15 --- [val.pmml.eval-1] log.sink : {"id":5199577666956545635,"deviceId":12,"value":1.599846306874403E-148,"accountId":-102,"timestamp":432345569956636875,"homeLocation":null,"homeLatitude":0.0,"homeLongitude":0.0,"distance":0.0,"_output":{"result":"3","type":"DISTANCE","entityRegistry":{"1":{"locator":null,"id":null,"name":"cluster_0","size":null,"extensions":[],"kohonenMap":null,"array":{"locator":null,"n":2,"type":"REAL","value":"27.685449231686356 3.349155420943788"},"partition":null,"covariances":null},"2":{"locator":null,"id":null,"name":"cluster_1","size":null,"extensions":[],"kohonenMap":null,"array":{"locator":null,"n":2,"type":"REAL","value":"32.16548772032409 2.544008885888239"},"partition":null,"covariances":null},"3":{"locator":null,"id":null,"name":"cluster_2","size":null,"extensions":[],"kohonenMap":null,"array":{"locator":null,"n":2,"type":"REAL","value":"27.691067777235084 1.7439567824479112"},"partition":null,"covariances":null}},"entity":{"locator":null,"id":null,"name":"cluster_2","size":null,"extensions":[],"kohonenMap":null,"array":{"locator":null,"n":2,"type":"REAL","value":"27.691067777235084 1.7439567824479112"},"partition":null,"covariances":null},"categoryValues":["1","2","3"],"entityIdRanking":["3","1","2"],"affinityRanking":[769.8366199024732,777.7009411939198,1041.090581497798],"entityAffinity":769.8366199024732,"displayValue":"cluster_2","entityId":"3"}} ----- +``` -Notice the added variable ___output__ and its __result__ mapping to a cluster number. You can also find information about the cluster mapping algorithm used (euclidean distance) and information about each cluster by their numbers. -*The value of ___output.result__ for each transaction is one of the clusters (starting at 1) defined at the trained model (see step 12 above)* +Notice the added variable `output` and its `result` mapping to a cluster number. You can also find information about the cluster mapping algorithm used (euclidean distance) and information about each cluster by their numbers. +*The value of `output.result` for each transaction is one of the clusters (starting at 1) defined at the trained model (see step 12 above)* ==== Deploying the third stream to SCDF - filtering high-risk transactions and inserting back to GemFire -20- Create and deploy the third stream to SCDF, called __result__ +20 - Create and deploy the third stream to SCDF, called `result` ----- +``` result = :eval.pmml > filter --expression=payload._output.result.toString().equals('2') | gemfire --region-name=Suspect --host-addresses=geode-server:10334 --keyExpression=payload.id.toString() ----- +``` Notice we're filtering only the transactions which are mapped to cluster number 3 in this example. Make sure you filter by a cluster number which is receiving a good amount of transactions, so you can show them in the TransactionsConsole app. Notice the new applications deployed to CF by this new stream. -21- Now run the TransactionsEmulator once more (check step 7) and check the TransactionsConsole UI. You should see some red/orange transactions in the map and their information on the associated box. +21 - Now run the `TransactionsEmulator` once more (check step 7) and check the `TransactionsConsole UI`. You should see some red/orange transactions in the map and their information on the associated box. In case you were successful until this point and haven't seen the flagged transactions coming in the UI: * Check the logs for the new two apps deployed by the third stream @@ -207,5 +240,34 @@ In case you were successful until this point and haven't seen the flagged transa image::fraud-detection.png[Demo Screenshot] +=== Appendices + +==== Appendix1: PostgreSQL Setup + +* Enable Postgres TCP/IP socket. Edit `postgresql.conf` and make sure `listen_addresses` is set to `*`. +* Enable client authentication. Eidt `pg_hba.conf` and add `host all all 192.168.11.11/32 trust` entry. +* Create user and database +Open psql as super user `psql -p5432` and run: +``` +CREATE USER pivotal WITH SUPERUSER LOGIN; +ALTER ROLE pivotal WITH PASSWORD 'pivotal'; +CREATE DATABASE gemfire; +GRANT ALL ON DATABASE gemfire TO pivotal; +``` + +* Edit the `model_postgres.sql` to set the absolute path to the CSV files. +``` +COPY transaction FROM '/FraudDetection-DataMicroservices/scripts/transaction.csv' DELIMITER ',' CSV HEADER; +COPY zip_codes FROM '/FraudDetection-DataMicroservices/scripts/zip_codes_states.csv' DELIMITER ',' CSV HEADER; +COPY pos_device FROM '/FraudDetection-DataMicroservices/scripts/pos_device.csv' DELIMITER ',' CSV HEADER; +``` + +* Create DDL and load the data +``` +cd scripts +psql -p5432 -U pivotal -d gemfire -f model_postgres.sql +``` + + diff --git a/TransactionsConsole/build.gradle b/TransactionsConsole/build.gradle index b81f3b8..3239f9a 100644 --- a/TransactionsConsole/build.gradle +++ b/TransactionsConsole/build.gradle @@ -66,4 +66,4 @@ dependencies { task wrapper(type: Wrapper) { gradleVersion = '2.3' -} +} \ No newline at end of file diff --git a/TransactionsConsole/gradle/wrapper/gradle-wrapper.jar b/TransactionsConsole/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000..30d399d Binary files /dev/null and b/TransactionsConsole/gradle/wrapper/gradle-wrapper.jar differ diff --git a/TransactionsConsole/gradle/wrapper/gradle-wrapper.properties b/TransactionsConsole/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..65d9746 --- /dev/null +++ b/TransactionsConsole/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Wed Sep 07 18:08:27 CEST 2016 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-2.4-bin.zip diff --git a/TransactionsConsole/manifest.yml b/TransactionsConsole/manifest.yml index 2db7a46..dd6baeb 100644 --- a/TransactionsConsole/manifest.yml +++ b/TransactionsConsole/manifest.yml @@ -1,7 +1,7 @@ --- applications: - name: webconsole - memory: 512M + memory: 256M instances: 1 host: transactions-console path: build/libs/WebConsole.jar diff --git a/TransactionsEmulator/build.gradle b/TransactionsEmulator/build.gradle index 55c49dc..8b2a788 100644 --- a/TransactionsEmulator/build.gradle +++ b/TransactionsEmulator/build.gradle @@ -1,49 +1,39 @@ buildscript { + ext { + springBootVersion = '1.4.0.RELEASE' + } repositories { mavenCentral() } dependencies { - classpath("org.springframework.boot:spring-boot-gradle-plugin:1.3.1.RELEASE") + classpath("org.springframework.boot:spring-boot-gradle-plugin:${springBootVersion}") } } apply plugin: 'spring-boot' apply plugin: 'java' - +apply plugin: 'maven' repositories { mavenCentral() "https://repository.apache.org/content/repositories/snapshots/" } - dependencies { - - compile "com.fasterxml.jackson.core:jackson-databind" compile "org.springframework.boot:spring-boot-autoconfigure" compile "org.springframework.boot:spring-boot-starter" - compile("org.springframework:spring-web") - compile 'org.mortbay.jetty:jetty-util:6.1.25'; - - testCompile("junit:junit:4.12") - - + compile "org.springframework:spring-web" + compile "org.mortbay.jetty:jetty-util:6.1.25" + testCompile "junit:junit:4.12" } -jar { - baseName='PoS_Emulator' - version = version -} - - -bootRepackage.withJarTask = jar - - -artifacts{ - archives jar -} +group = 'io.pivotal.demo' +archivesBaseName = 'PoS_Emulator' +version = '1.0.0.BUILD-SNAPSHOT' task wrapper(type: Wrapper) { gradleVersion = '2.3' } + +install.dependsOn(bootRepackage) \ No newline at end of file diff --git a/TransactionsEmulator/gradle/wrapper/gradle-wrapper.jar b/TransactionsEmulator/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000..30d399d Binary files /dev/null and b/TransactionsEmulator/gradle/wrapper/gradle-wrapper.jar differ diff --git a/TransactionsEmulator/gradle/wrapper/gradle-wrapper.properties b/TransactionsEmulator/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..65d9746 --- /dev/null +++ b/TransactionsEmulator/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Wed Sep 07 18:08:27 CEST 2016 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-2.4-bin.zip diff --git a/TransactionsEmulator/src/main/java/io/pivotal/demo/sko/Emulator.java b/TransactionsEmulator/src/main/java/io/pivotal/demo/sko/Emulator.java index 23331f9..5c37c86 100644 --- a/TransactionsEmulator/src/main/java/io/pivotal/demo/sko/Emulator.java +++ b/TransactionsEmulator/src/main/java/io/pivotal/demo/sko/Emulator.java @@ -155,7 +155,7 @@ private Long getHomePoS(Long accountId){ private void loadPoSCounties() throws IOException { counties = new ArrayList(); - InputStream is = ClassLoader.getSystemResourceAsStream("counties.csv"); + InputStream is = ClassLoader.getSystemResourceAsStream("BOOT-INF/classes/counties.csv"); BufferedReader br = new BufferedReader(new InputStreamReader(is)); String line = br.readLine(); //skip header diff --git a/scripts/scdf-stream-apps.properties b/scripts/scdf-stream-apps.properties deleted file mode 100644 index e993258..0000000 --- a/scripts/scdf-stream-apps.properties +++ /dev/null @@ -1,46 +0,0 @@ -source.file=maven://org.springframework.cloud.stream.app:file-source-rabbit:1.0.0.BUILD-SNAPSHOT -source.ftp=maven://org.springframework.cloud.stream.app:ftp-source-rabbit:1.0.0.BUILD-SNAPSHOT -source.gemfire=maven://org.springframework.cloud.stream.app:gemfire-source-rabbit:1.0.0.M1 -source.gemfire-cq=maven://org.springframework.cloud.stream.app:gemfire-cq-source-rabbit:1.0.0.M1 -source.jdbc=maven://org.springframework.cloud.stream.app:jdbc-source-rabbit:1.0.0.BUILD-SNAPSHOT -source.jms=maven://org.springframework.cloud.stream.app:jms-source-rabbit:1.0.0.BUILD-SNAPSHOT -source.http=maven://org.springframework.cloud.stream.app:http-source-rabbit:1.0.0.BUILD-SNAPSHOT -source.load-generator=maven://org.springframework.cloud.stream.app:load-generator-source-rabbit:1.0.0.BUILD-SNAPSHOT -source.mail=maven://org.springframework.cloud.stream.app:mail-source-rabbit:1.0.0.BUILD-SNAPSHOT -source.rabbit=maven://org.springframework.cloud.stream.app:rabbit-source-rabbit:1.0.0.BUILD-SNAPSHOT -source.s3=maven://org.springframework.cloud.stream.app:s3-source-rabbit:1.0.0.BUILD-SNAPSHOT -source.sftp=maven://org.springframework.cloud.stream.app:sftp-source-rabbit:1.0.0.BUILD-SNAPSHOT -source.syslog=maven://org.springframework.cloud.stream.app:syslog-source-rabbit:1.0.0.M1 -source.tcp=maven://org.springframework.cloud.stream.app:tcp-source-rabbit:1.0.0.BUILD-SNAPSHOT -source.time=maven://org.springframework.cloud.stream.app:time-source-rabbit:1.0.0.BUILD-SNAPSHOT -source.trigger=maven://org.springframework.cloud.stream.app:trigger-source-rabbit:1.0.0.M1 -source.triggertask=maven://org.springframework.cloud.stream.app:triggertask-source-rabbit:1.0.0.BUILD-SNAPSHOT -source.twitterstream=maven://org.springframework.cloud.stream.app:twitterstream-source-rabbit:1.0.0.BUILD-SNAPSHOT -processor.bridge=maven://org.springframework.cloud.stream.app:bridge-processor-rabbit:1.0.0.BUILD-SNAPSHOT -processor.filter=maven://org.springframework.cloud.stream.app:filter-processor-rabbit:1.0.0.BUILD-SNAPSHOT -processor.groovy-filter=maven://org.springframework.cloud.stream.app:groovy-filter-processor-rabbit:1.0.0.BUILD-SNAPSHOT -processor.groovy-transform=maven://org.springframework.cloud.stream.app:groovy-transform-processor-rabbit:1.0.0.BUILD-SNAPSHOT -processor.httpclient=maven://org.springframework.cloud.stream.app:httpclient-processor-rabbit:1.0.0.BUILD-SNAPSHOT -processor.pmml=maven://org.springframework.cloud.stream.app:pmml-processor-rabbit:1.0.0.M1 -processor.scriptable-transform=maven://org.springframework.cloud.stream.app:scriptable-transform-processor-rabbit:1.0.0.BUILD-SNAPSHOT -processor.splitter=maven://org.springframework.cloud.stream.app:splitter-processor-rabbit:1.0.0.BUILD-SNAPSHOT -processor.transform=maven://org.springframework.cloud.stream.app:transform-processor-rabbit:1.0.0.BUILD-SNAPSHOT -sink.aggregate-counter=maven://org.springframework.cloud.stream.app:aggregate-counter-sink-rabbit:1.0.0.BUILD-SNAPSHOT -sink.cassandra=maven://org.springframework.cloud.stream.app:cassandra-sink-rabbit:1.0.0.M1 -sink.counter=maven://org.springframework.cloud.stream.app:counter-sink-rabbit:1.0.0.BUILD-SNAPSHOT -sink.field-value-counter=maven://org.springframework.cloud.stream.app:field-value-counter-sink-rabbit:1.0.0.BUILD-SNAPSHOT -sink.file=maven://org.springframework.cloud.stream.app:file-sink-rabbit:1.0.0.BUILD-SNAPSHOT -sink.ftp=maven://org.springframework.cloud.stream.app:ftp-sink-rabbit:1.0.0.BUILD-SNAPSHOT -sink.gemfire=maven://org.springframework.cloud.stream.app:gemfire-sink-rabbit:1.0.0.M1 -sink.gpfdist=maven://org.springframework.cloud.stream.app:gpfdist-sink-rabbit:1.0.0.M1 -sink.hdfs=maven://org.springframework.cloud.stream.app:hdfs-sink-rabbit:1.0.0.BUILD-SNAPSHOT -sink.hdfs-dataset=maven://org.springframework.cloud.stream.app:hdfs-dataset-sink-rabbit:1.0.0.M1 -sink.jdbc=maven://org.springframework.cloud.stream.app:jdbc-sink-rabbit:1.0.0.BUILD-SNAPSHOT -sink.log=maven://org.springframework.cloud.stream.app:log-sink-rabbit:1.0.0.BUILD-SNAPSHOT -sink.rabbit=maven://org.springframework.cloud.stream.app:rabbit-sink-rabbit:1.0.0.BUILD-SNAPSHOT -sink.redis-pubsub=maven://org.springframework.cloud.stream.app:redis-sink-rabbit:1.0.0.M1 -sink.router=maven://org.springframework.cloud.stream.app:router-sink-rabbit:1.0.0.BUILD-SNAPSHOT -sink.tcp=maven://org.springframework.cloud.stream.app:tcp-sink-rabbit:1.0.0.BUILD-SNAPSHOT -sink.throughput=maven://org.springframework.cloud.stream.app:throughput-sink-rabbit:1.0.0.BUILD-SNAPSHOT -sink.websocket=maven://org.springframework.cloud.stream.app:websocket-sink-rabbit:1.0.0.BUILD-SNAPSHOT - diff --git a/start-scdf-pcfdev.sh b/start-scdf-pcfdev.sh index 434da7d..1fc0615 100755 --- a/start-scdf-pcfdev.sh +++ b/start-scdf-pcfdev.sh @@ -1,18 +1,23 @@ -SCDF_BUILD_PATH=/Users/fmelo/scdf - +#!/bin/bash +#SCDF_BUILD_PATH=/Users/fmelo/scdf export SPRING_CLOUD_DEPLOYER_CLOUDFOUNDRY_URL=http://api.local.pcfdev.io export SPRING_CLOUD_DEPLOYER_CLOUDFOUNDRY_ORG=pcfdev-org export SPRING_CLOUD_DEPLOYER_CLOUDFOUNDRY_SPACE=pcfdev-space export SPRING_CLOUD_DEPLOYER_CLOUDFOUNDRY_DOMAIN=local.pcfdev.io -export SPRING_CLOUD_DEPLOYER_CLOUDFOUNDRY_SERVICES=rabbit,redis +export SPRING_CLOUD_DEPLOYER_CLOUDFOUNDRY_STREAM_SERVICES=rabbit,redis export SPRING_CLOUD_DEPLOYER_CLOUDFOUNDRY_USERNAME=admin export SPRING_CLOUD_DEPLOYER_CLOUDFOUNDRY_PASSWORD=admin export SPRING_CLOUD_DEPLOYER_CLOUDFOUNDRY_SKIP_SSL_VALIDATION=true -export SPRING_CLOUD_DEPLOYER_CLOUDFOUNDRY_MEMORY=512 -export SPRING_CLOUD_DEPLOYER_CLOUDFOUNDRY_DISK=512 -export SPRING_CLOUD_DEPLOYER_CLOUDFOUNDRY_BUILDPACK=java_buildpack -export SPRING_CLOUD_DEPLOYER_CLOUDFOUNDRY_ENABLE_RANDOM_APP_NAME_PREFIX=false -export MAVEN_OFFLINE=true -java -jar $SCDF_BUILD_PATH/spring-cloud-dataflow-server-cloudfoundry-1.0.0.BUILD-SNAPSHOT.jar +export SPRING_CLOUD_DEPLOYER_CLOUDFOUNDRY_STREAM_MEMORY=384 +export SPRING_CLOUD_DEPLOYER_CLOUDFOUNDRY_STREAM_DISK=512 + +export SPRING_CLOUD_DEPLOYER_CLOUDFOUNDRY_STREAM_ENABLE_RANDOM_APP_NAME_PREFIX=false +export SPRING_CLOUD_DEPLOYER_CLOUDFOUNDRY_STREAM_BUILDPACK=java_buildpack + +# export MAVEN_OFFLINE=true +export MAVEN_REMOTE_REPOSITORIES_REPO1_URL=https://repo.spring.io/release + +java -jar ./spring-cloud-dataflow-server-cloudfoundry-1.0.0.RELEASE.jar +