本篇文章为大家展示了怎样使用sbt构建spark的项目，内容简明扼要并且容易理解，绝对能使你眼前一亮，通过这篇文章的详细介绍希望你能有所收获。
用Intellij 构建sbt项目 scala 使用2.10.4
计算
安全
数据库
网络和加速
企业服务
name := "gstorm"
version := "1.0"
version := "1.0"
//Older Scala Version
scalaVersion := "2.10.4"
val overrideScalaVersion = "2.11.8"
val sparkVersion = "2.0.0"
val sparkXMLVersion = "0.3.3"
val sparkCsvVersion = "1.4.0"
val sparkElasticVersion = "2.3.4"
val sscKafkaVersion = "2.0.1"
val sparkMongoVersion = "1.0.0"
val sparkCassandraVersion = "1.6.0"
//Override Scala Version to the above 2.11.8 version
ivyScala := ivyScala.value map {
_.copy(overrideScalaVersion = true)
}
resolvers ++= Seq(
"All Spark Repository -> bintray-spark-packages" at "https://dl.bintray.com/spark-packages/maven/"
)
libraryDependencies ++= Seq(
"org.apache.spark" %% "spark-core" % sparkVersion exclude("jline", "2.12"),
"org.apache.spark" %% "spark-sql" % sparkVersion excludeAll(ExclusionRule(organization = "jline"), ExclusionRule("name", "2.12")),
"org.apache.spark" %% "spark-hive" % sparkVersion,
"org.apache.spark" %% "spark-yarn" % sparkVersion,
"com.databricks" %% "spark-xml" % sparkXMLVersion,
"com.databricks" %% "spark-csv" % sparkCsvVersion,
"org.apache.spark" %% "spark-graphx" % sparkVersion,
"org.apache.spark" %% "spark-catalyst" % sparkVersion,
"org.apache.spark" %% "spark-streaming" % sparkVersion,
// "com.101tec" % "zkclient" % "0.9",
"org.elasticsearch" %% "elasticsearch-spark" % sparkElasticVersion,
// "org.apache.spark" %% "spark-streaming-kafka-0-10_2.11" % sscKafkaVersion,
"org.mongodb.spark" % "mongo-spark-connector_2.11" % sparkMongoVersion,
"com.stratio.datasource" % "spark-mongodb_2.10" % "0.11.1",
"dibbhatt" % "kafka-spark-consumer" % "1.0.8",
"net.liftweb" %% "lift-webkit" % "2.6.2"
)WordCount.scala
import org.apache.spark.sql.SparkSession
object WordCount {
def main(args: Array[String]): Unit = {
val spark = SparkSession
.builder()
.appName("Spark SQL Example")
.master("local[2]")
.config("spark.sql.codegen.WordCount", "true")
.getOrCreate()
val sc = spark.sparkContext
val textFile = sc.textFile("hdfs://hadoop:9000/words.txt")
val wordCounts = textFile.flatMap(line => line.split(" ")).map(word => (word, 1)).reduceByKey((a, b) => a + b)
wordCounts.collect.foreach(println)
}
}上述内容就是怎样使用sbt构建spark的项目，你们学到知识或技能了吗？如果还想学到更多技能或者丰富自己的知识储备，欢迎关注亿速云行业资讯频道。
亿速云「云服务器」，即开即用、新一代英特尔至强铂金CPU、三副本存储NVMe SSD云盘，价格低至29元/月。点击查看>>
免责声明：本站发布的内容（图片、视频和文字）以原创、转载和分享为主，文章观点不代表本网站立场，如果涉及侵权请联系站长邮箱：is@yisu.com进行举报，并提供相关证据，一经查实，将立刻删除涉嫌侵权内容。
原文链接：https://my.oschina.net/lh7923495/blog/745939
