Spark Streaming 监控HDFS目录

xiaoxiao2021-02-28  6

package org.lm.spark.streaming import org.apache.spark.SparkConf import org.apache.spark.streaming.{Seconds, StreamingContext} object StreamingWordCountOnLine { def main(args: Array[String]): Unit = { val conf=new SparkConf().setAppName("Streaming Word Count OnLine").setMaster("spark://192.168.189.128:7077") val ssc=new StreamingContext(conf,Seconds(10)) val lines=ssc.textFileStream("hdfs://192.168.189.128:9000/user/StreamingText") val words=lines.flatMap(_.split(" ")) val pairs=words.map(word=>(word,1)) val wordcounts=pairs.reduceByKey(_+_) wordcounts.print() ssc.start() ssc.awaitTermination() } }
转载请注明原文地址: https://www.6miu.com/read-2000091.html

最新回复(0)