diff --git a/README.md b/README.md index 72e2c31..da10e38 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ sbt test ## Running the application in local mode ``` -spark-submit --class com.cloudwick.spark.WordCountRunner \ +spark-submit --class com.careerbuilder.spark.WordCountRunner \ --master "local[*]" \ target/scala-2.10/spark-starter_2.10-1.0.jar \ [input_path] \ @@ -44,7 +44,7 @@ spark-submit --class com.cloudwick.spark.WordCountRunner \ ## Running the application in the cluster ``` -spark-submit --class com.cloudwick.spark.WordCountRunner \ +spark-submit --class com.careerbuilder.spark.WordCountRunner \ --master "local[*]" \ [path_to_jar]/spark-starter_2.10-1.0.jar \ [input_path] \ diff --git a/build.sbt b/build.sbt index edbdb3b..d2487b4 100644 --- a/build.sbt +++ b/build.sbt @@ -1,4 +1,4 @@ -name := "spark-starter" +name := "Carotene4" version := "1.0" @@ -8,7 +8,9 @@ resolvers ++= Seq( "typesafe-repository" at "http://repo.typesafe.com/typesafe/releases" ) -val sparkVersion = "1.2.1" +resolvers += "Artima Maven Repository" at "http://repo.artima.com/releases" + +val sparkVersion = "1.6.0" // build a uber jar if using any of the external streaming components libraryDependencies ++= Seq( @@ -18,11 +20,20 @@ libraryDependencies ++= Seq( "org.apache.spark" %% "spark-streaming-kafka" % sparkVersion, "org.apache.spark" %% "spark-streaming-twitter" % sparkVersion, // Test dependencies - "org.scalatest" %% "scalatest" % "2.2.4" % "test", +// "org.scalatest" %% "scalatest" % "2.2.4" % "test", + "org.scalactic" %% "scalactic" % "3.0.0", + "org.scalatest" %% "scalatest" % "3.0.0" % "test", "org.xerial.snappy" % "snappy-java" % "1.1.1.7" -) + ) + +assemblyMergeStrategy in assembly := { + case PathList("org", "apache", "spark", "unused", "UnusedStubClass.class") => MergeStrategy.discard + case x => + val oldStrategy = (assemblyMergeStrategy in assembly).value + oldStrategy(x) +} parallelExecution in Test := false fork in Test := true - \ No newline at end of file + diff --git a/project/assembly.sbt b/project/assembly.sbt index 20861f2..1741de9 100644 --- a/project/assembly.sbt +++ b/project/assembly.sbt @@ -1 +1 @@ -addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.13.0") \ No newline at end of file +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.3") \ No newline at end of file diff --git a/project/plugins.sbt b/project/plugins.sbt index 14a6ca1..98ea2e9 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1 +1,4 @@ -logLevel := Level.Warn \ No newline at end of file +logLevel := Level.Warn +resolvers += "Artima Maven Repository" at "http://repo.artima.com/releases" + +addSbtPlugin("com.artima.supersafe" % "sbtplugin" % "1.1.0") \ No newline at end of file diff --git a/src/main/scala/com/cloudwick/spark/WordCount.scala b/src/main/scala/com/careerbuilder/spark/WordCount.scala similarity index 93% rename from src/main/scala/com/cloudwick/spark/WordCount.scala rename to src/main/scala/com/careerbuilder/spark/WordCount.scala index 638b3cd..708e59e 100644 --- a/src/main/scala/com/cloudwick/spark/WordCount.scala +++ b/src/main/scala/com/careerbuilder/spark/WordCount.scala @@ -1,11 +1,11 @@ -package com.cloudwick.spark +package com.careerbuilder.spark import org.apache.spark.SparkContext._ import org.apache.spark.rdd.RDD /** * Simple word count application to illustrate spark standalone applications usage - * @author ashrith + * @author oozturk */ case class WordCount(word: String, count: Int) diff --git a/src/main/scala/com/cloudwick/spark/WordCountRunner.scala b/src/main/scala/com/careerbuilder/spark/WordCountRunner.scala similarity index 92% rename from src/main/scala/com/cloudwick/spark/WordCountRunner.scala rename to src/main/scala/com/careerbuilder/spark/WordCountRunner.scala index c41b2b8..ec2326f 100644 --- a/src/main/scala/com/cloudwick/spark/WordCountRunner.scala +++ b/src/main/scala/com/careerbuilder/spark/WordCountRunner.scala @@ -1,10 +1,10 @@ -package com.cloudwick.spark +package com.careerbuilder.spark import org.apache.spark.{SparkContext, SparkConf, Logging} /** * Main method to the Spark WordCount application. - * @author ashrith + * @author oozturk */ object WordCountRunner extends App with Logging { if (args.length < 2) { diff --git a/src/main/scala/com/cloudwick/spark/package.scala b/src/main/scala/com/careerbuilder/spark/package.scala similarity index 82% rename from src/main/scala/com/cloudwick/spark/package.scala rename to src/main/scala/com/careerbuilder/spark/package.scala index 1740fe5..a95c5e4 100644 --- a/src/main/scala/com/cloudwick/spark/package.scala +++ b/src/main/scala/com/careerbuilder/spark/package.scala @@ -1,8 +1,8 @@ -package com.cloudwick +package com.careerbuilder /** * Pakcage object for examples - * @author ashrith + * @author oozturk */ package object spark { implicit class StringUtils(val value: String) { diff --git a/src/test/scala/com/cloudwick/spark/WordCountSpec.scala b/src/test/scala/com/careerbuilder/spark/WordCountSpec.scala similarity index 92% rename from src/test/scala/com/cloudwick/spark/WordCountSpec.scala rename to src/test/scala/com/careerbuilder/spark/WordCountSpec.scala index 8903b11..0f9d3a8 100644 --- a/src/test/scala/com/cloudwick/spark/WordCountSpec.scala +++ b/src/test/scala/com/careerbuilder/spark/WordCountSpec.scala @@ -1,6 +1,6 @@ -package com.cloudwick.spark +package com.careerbuilder.spark -import com.cloudwick.spark.sparkspec.SparkSpec +import com.careerbuilder.spark.WordCount import org.scalatest.{FlatSpec, GivenWhenThen, Matchers} /** diff --git a/src/test/scala/com/cloudwick/spark/sparkspec/SparkSpec.scala b/src/test/scala/com/careerbuilder/spark/sparkspec/SparkSpec.scala similarity index 95% rename from src/test/scala/com/cloudwick/spark/sparkspec/SparkSpec.scala rename to src/test/scala/com/careerbuilder/spark/sparkspec/SparkSpec.scala index 906f4da..69bd0b5 100644 --- a/src/test/scala/com/cloudwick/spark/sparkspec/SparkSpec.scala +++ b/src/test/scala/com/careerbuilder/spark/sparkspec/SparkSpec.scala @@ -1,4 +1,4 @@ -package com.cloudwick.spark.sparkspec +package com.careerbuilder.spark import org.apache.spark.{SparkConf, SparkContext} import org.scalatest.{Suite, BeforeAndAfterAll}