Spark入门运行wordcount

        2.11.8
        2.2.2
        2.7.6
    
    
        spark-core-study
        spark-common
    
    
        
            
                junit
                junit
                4.12
            

            
            
            
                org.apache.spark
                spark-core_2.11
                ${spark.version}
            
            
            
                org.apache.spark
                spark-sql_2.11
                ${spark.version}
            
            
            
                org.apache.spark
                spark-streaming_2.11
                ${spark.version}
            
        
    

        
            org.apache.spark
            spark-core_2.11
            
            provided
        

        
    

package blog

import org.apache.log4j.{Level, Logger}
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

/**
  * WordCount
  */
object WordCount {
  def main(args: Array[String]): Unit = {
    //提示语，如果没有参数就报错，直接退出jvm
    if (args == null || args.length 
          |input: input file path
        """.stripMargin
        //stripMargin是用来分行显示的
      )
      System.exit(-1)
    }
    val Array(input) = args
    //打印需要的日志
    Logger.getLogger("org.apache.hadoop").setLevel(Level.INFO)
    Logger.getLogger("org.apache.spark").setLevel(Level.INFO)
    Logger.getLogger("org.spark_project").setLevel(Level.INFO)
    val conf = new SparkConf()
      //这里名字可以随便去
      .setAppName("WordCount")
    //加上下面这句话就是在单个节点跑，Web UI上不会有显示
    //.setMaster("local[*]")
    //SparkContext为spark的入口
    val sc = new SparkContext(conf)
    //获取每一行
    val lines: RDD[String] = sc.textFile(input)
    //这里我们可以看一下spark将文件分成了几个区
    println("##############################partition num of lines is:" + lines.getNumPartitions)
    //flatMap进行过滤，s+表示切到空格以及多个空格
    val words: RDD[String] = lines.flatMap(line => line.split("s+"))
    //map端直接调用
    val pairs: RDD[(String, Int)] = words.map(word => count(word))
    //将出现多次的结果在reduce端相加
    val retRDD: RDD[(String, Int)] = pairs.reduceByKey((v1, v2) => v1 + v2)
    //控制台打印
    retRDD
      .collect() //
      .foreach(println)
    sc.stop()

  }

  //默认出现一次的单词为word, 1
  def count(word: String): (String, Int) = (word, 1)
}

vi spark-submit-wc-standalone.sh

#!/bin/sh

SPARK_HOME=/home/hadoop/apps/spark

${SPARK_HOME}/bin/spark-submit 
--class blog.WordCount 
--master spark://hadoop01:7077 
--deploy-mode client 
--total-executor-cores 2 
--executor-cores 1 
--executor-memory 600M 
/home/hadoop/jars/spark/spark-wc.jar 
hdfs://bd服务器托管网1906/data/spark/hello.txt

hello you
hello you
hello me
hello   you
hello you
hello me
This page outlines the steps for getting a Storm cluster up and running

hdfs dfs -put hello.txt /data/spark/hello.txt

./spark-submit-wc-standalone.sh

chmod 777 ./spark-submit-wc-standalone.sh