# 从零开始学习Spark--第3章 spark的example

2017/01/17 09:37

1. 计算Pi值

1.1 命令行：

#spark安装在/home/brian/setup/spark-new/spark-0.8.0目录
cd /home/brian/setup/spark-new/spark-0.8.0

#在本地运行计算Pi值的Spark程序，4个工作线程
./run-example org.apache.spark.examples.SparkPi local[4]

1.2 Web UI

1.3 SparkPi.scal的源代码分析

1.3.1 代码在examples/src/main/scala/org/apache/spark/examples目录下

1.3.2 代码内容如下：
package org.apache.spark.examples

import scala.math.random
import org.apache.spark._
import SparkContext._

/** Computes an approximation to pi */
object SparkPi {
def main(args: Array[String]) {
if (args.length == 0) {
System.err.println("Usage: SparkPi <master> [<slices>]")
System.exit(1)
}
val spark = new SparkContext(args(0), "SparkPi",
System.getenv("SPARK_HOME"), Seq(System.getenv("SPARK_EXAMPLES_JAR")))
val slices = if (args.length > 1) args(1).toInt else 2
val n = 100000 * slices
val count = spark.parallelize(1 to n, slices).map { i =>
val x = random * 2 - 1
val y = random * 2 - 1
if (x*x + y*y < 1) 1 else 0
}.reduce(_ + _)
println("Pi is roughly " + 4.0 * count / n)
System.exit(0)
}
}

val count = spark.parallelize(1 to n, slices).map { i =>
val x = random * 2 - 1
val y = random * 2 - 1
if (x*x + y*y < 1) 1 else 0
}.reduce(_ + _)

2. LocalPi.scala

4. CassandraTest.scala

5. ExceptionHandlingTest.scala

6. GroupByTest.scala

7. HBaseTest.scala

8. HdfsTest.scala

9. LocalALS.scala
ALS是交替最小二乘法，通常用于推荐系统的算法。

10. LocalFileLR.scala

11. LocalKMeans.scala
K-Means聚类算法。

12. LocalLR.scala

13. LogQuery.scala

....

0
0 收藏

0 评论
0 收藏
0