Hadoop天气系统

1.写完计数程序打包成jar
只要class文件即可
2.上传到node1上
3.hadoop jar weather.jar com.hadoop.mr.weather.WeatherSystem

hdfs dfs -ls /data/weather/output
hdfs dfs -cat /data/weather/output/part-r-00000
也可以把内容copy到当前的目录
hdfs dfs -get /data/weather/output/* ./

public class WeatherSystem {public static void main(String[] args) throws Exception {Configuration configuration = new Configuration(true);Job job = Job.getInstance(configuration);job.setJarByClass(WeatherSystem.class);job.setJobName("weather");//map startjob.setMapperClass(WeatherMapper.class);job.setMapOutputKeyClass(WeatherData.class);job.setOutputValueClass(IntWritable.class);job.setPartitionerClass(WeatherPartition.class);job.setSortComparatorClass(WeatherComparator.class);
//      job.setCombinerClass(cls);//map end//reduce startjob.setGroupingComparatorClass(WeatherGroupComparator.class);job.setReducerClass(WeatherReducer.class);job.setNumReduceTasks(2);//reduce endPath input = new Path("/data/weather/input/weather.txt");FileInputFormat.addInputPath(job, input );Path output = new Path("/data/weather/output");//测试专用为防止出问题if(output.getFileSystem(configuration).exists(output)){output.getFileSystem(configuration).delete(output, true);}FileOutputFormat.setOutputPath(job, output);  // Submit the job, then poll for progress until the job is completejob.waitForCompletion(true);}}

public class WeatherMapper extends Mapper<LongWritable, Text, WeatherData, IntWritable>{private WeatherData wdkey = new WeatherData();private final static IntWritable ivalue = new IntWritable(1);public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {String str[] = StringUtils.split(value.toString(),'\t');try{SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");Date date = sdf.parse(str[0]);Calendar calendar = Calendar.getInstance();calendar.setTime(date);wdkey.setYear(calendar.get(calendar.YEAR));wdkey.setMonth(calendar.get(calendar.MONTH)+1);wdkey.setDay(calendar.get(calendar.DAY_OF_MONTH));int temperature = Integer.valueOf(str[1].substring(0, str[1].length()-1));wdkey.setTemperature(temperature);ivalue.set(temperature);context.write(wdkey, ivalue);} catch (Exception e) {// TODO: handle exception}}}

public class WeatherReducer extends Reducer<WeatherData, IntWritable, Text, IntWritable>{//相同的key为一组//1999 01 01 38(key)  38(value)//1999 01 11 32(key)  32(value)//1999 01 12 38(key)  38(value)//1999 01 11 28(key)  28(value)private Text key = new Text();private IntWritable result = new IntWritable();@Overrideprotected void reduce(WeatherData data, Iterable<IntWritable> iterable,Reducer<WeatherData, IntWritable, Text, IntWritable>.Context context) throws IOException, InterruptedException {// TODO Auto-generated method stubint flag = 0;int day = 0;for(IntWritable writable : iterable){if(flag == 0){//1999-01-01 38key.set(data.getYear()+"-"+data.getMonth()+"-"+data.getDay()+" " + data.getTemperature());result.set(data.getTemperature());flag ++;day = data.getDay();context.write(key, result);}if(flag != 0 && day != data.getDay()){key.set(data.getYear()+"-"+data.getMonth()+"-"+data.getDay()+" " + data.getTemperature());result.set(data.getTemperature());context.write(key, result);break;}}}}

public class WeatherPartition extends Partitioner<WeatherData, IntWritable>{@Overridepublic int getPartition(WeatherData key, IntWritable value, int numPartitions) {// 这里对key做分组，这个方法应该简单，尽量让reduce能并行处理完//针对weather没有效果return key.hashCode() % numPartitions;}}

import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;public class WeatherComparator extends WritableComparator{public WeatherComparator(){super(WeatherData.class, true);}@Overridepublic int compare(WritableComparable a, WritableComparable b) {// TODO Auto-generated method stubWeatherData data1 = (WeatherData)a;WeatherData data2 = (WeatherData)b;int yearCompare = Integer.compare(data1.getYear(), data2.getYear());if(yearCompare == 0){int monthCompare = Integer.compare(data1.getMonth(), data2.getMonth());if(monthCompare == 0){int temperatureCompare = Integer.compare(data1.getTemperature(), data2.getTemperature());return -temperatureCompare;}return monthCompare;}return yearCompare;}}

package com.hadoop.mr.weather;import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;import org.apache.hadoop.io.WritableComparable;public class WeatherData implements WritableComparable<WeatherData>{private int year;private int month;private int day;private int temperature;@Overridepublic void write(DataOutput out) throws IOException {// TODO Auto-generated method stubout.writeInt(year);out.writeInt(month);out.writeInt(day);out.writeInt(temperature);}@Overridepublic void readFields(DataInput in) throws IOException {// TODO Auto-generated method stubyear = in.readInt();month = in.readInt();day = in.readInt();temperature = in.readInt();}@Overridepublic int compareTo(WeatherData that) {// TODO Auto-generated method stub//日期正序int yearCompare = Integer.compare(this.year, that.year);if(yearCompare == 0){int monthCompare = Integer.compare(this.month, that.month);if(monthCompare == 0 ){return Integer.compare(this.day, that.day);}return monthCompare;}return yearCompare;}/*** @return the year*/public int getYear() {return year;}/*** @param year the year to set*/public void setYear(int year) {this.year = year;}/*** @return the month*/public int getMonth() {return month;}/*** @param month the month to set*/public void setMonth(int month) {this.month = month;}/*** @return the day*/public int getDay() {return day;}/*** @param day the day to set*/public void setDay(int day) {this.day = day;}/*** @return the temperature*/public int getTemperature() {return temperature;}/*** @param temperature the temperature to set*/public void setTemperature(int temperature) {this.temperature = temperature;}}

import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;public class WeatherGroupComparator extends WritableComparator{public WeatherGroupComparator(){super(WeatherData.class, true);}@Overridepublic int compare(WritableComparable a, WritableComparable b) {// TODO Auto-generated method stubWeatherData data1 = (WeatherData)a;WeatherData data2 = (WeatherData)b;int yearCompare = Integer.compare(data1.getYear(), data2.getYear());if(yearCompare == 0){return Integer.compare(data1.getMonth(), data2.getMonth());}return yearCompare;}
}

Hadoop天气系统相关推荐

hadoop 添加删除机器以及设置免密登录
添加hadoop机器先在slaves中添加机器然后启动datanode $: ./usr/hadoop-0.20.2-cdh3u4/bin/hadoop-daemon.sh start datan ...
linux环境下快速配置hadoop集群免密登录
背景在hadoop的日常使用过程中经常需要登录某些机器,如何更好的免密登录呢?这将为我们节省大量的时间操作假设你需要在A机器上免密登录B机器,那么你首先要确定B机器下是有秘钥文件的.如何确定是否 ...
hadoop问题小结
20220322 https://blog.csdn.net/lt5227/article/details/119459827 hadoop控制台设置密码访问验证 20220314 进入hive 高 ...
hadoop,spark,scala,flink 大数据分布式系统汇总
20220314 https://shimo.im/docs/YcPW8YY3T6dT86dV/read 尚硅谷大数据文档资料 iceberg相当于对hive的读写,starrocks相当于对mysq ...
spark,hadoop区别
https://zhuanlan.zhihu.com/p/95016937 Spark和Hadoop的区别和比较: 1.原理比较: Hadoop和Spark都是并行计算,两者都是用MR模型进行计算 H ...
2021年大数据Hadoop（三十）：Hadoop3.x的介绍
全网最详细的Hadoop文章系列,强烈建议收藏加关注! 后面更新文章都会列出历史文章目录,帮助大家回顾知识重点. 目录本系列历史文章前言 Hadoop3.x的介绍介绍 Hadoop 3.0新特性 ...
2021年大数据Hadoop（二十九）：关于YARN常用参数设置
全网最详细的Hadoop文章系列,强烈建议收藏加关注! 后面更新文章都会列出历史文章目录,帮助大家回顾知识重点. 目录本系列历史文章前言关于yarn常用参数设置设置container分配最小内 ...
2021年大数据Hadoop（二十七）：YARN运行流程
全网最详细的Hadoop文章系列,强烈建议收藏加关注! 后面更新文章都会列出历史文章目录,帮助大家回顾知识重点. 目录本系列历史文章前言 Yarn运行流程本系列历史文章 2021年大数据Hado ...
2021年大数据Hadoop（二十六）：YARN三大组件介绍
全网最详细的Hadoop文章系列,强烈建议收藏加关注! 后面更新文章都会列出历史文章目录,帮助大家回顾知识重点. 目录本系列历史文章前言 Yarn三大组件介绍 ResourceManager No ...

Hadoop天气系统

Hadoop天气系统相关推荐

最新文章

热门文章