《使用MapReduce分析数据》由会员分享，可在线阅读，更多相关《使用MapReduce分析数据（14页珍藏版）》请在金锄头文库上搜索。
1、云计算及应用实验报告 项目名称 使用MapReduce分析数据 专业班级 学 号 姓 名 实验成绩：批阅教师：2016年6 月3 日实验3使用MapReduce分析数据实验学时： 2 实验地点： 实验日期： 一、实验目的1.掌握MapReduce的使用方法2.学会使用MapReduce分析数据二、实验内容1.运行MapReduce的wordcount实例2.使用MapReduce完成二度朋友的实验三、实验方法包括实验方法、原理、技术、方案等。四、实验步骤1. 打开hadoop的各个节点2. 运行hadoop中的wordcount实例3. 编写并编译二度朋友实验的代码4. 将要处理的数据传送到h
2、adoop中5. 运行代码处理数据。五、实验结果记录实验输出数据和结果。六、实验结论1. 创建wordcount_in文件夹2. 创建text.txt文件用来保存要处理的数据3. 将要处理的文件text.txt上传到hadoop4. 运行hadoop-example实例5 查看运行的结果6. 编写二度朋友实验的代码import java.io.IOException;import java.util.Random;import java.util.Vector;import org.apache.hadoop.conf.Configuration;import org.apache.hadoo
3、p.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduc
4、e.lib.output.FileOutputFormat;import org.apache.hadoop.util.GenericOptionsParser;public class deg2friend public static class job1Mapper extends Mapperprivate Text job1map_key = new Text();private Text job1map_value = new Text(); public void map(Object key, Text value, Context context) throws IOExcep
5、tion, InterruptedException String eachterm = value.toString().split(,|t); for(int i=1; ieachterm.length; i+) if(pareTo(eachtermi)0) job1map_value.set(eachtermi+t+eachterm0); job1map_key.set(eachterm0); context.write(job1map_key, job1map_value); job1map_key.set(eachtermi); context.write(job1map_key,
6、job1map_value); public static class job1Reducer extends Reducer private Text job1reduce_key = new Text();private Text job1reduce_value = new Text();public void reduce(Text key, Iterable values, Context context) throws IOException, InterruptedException String someperson = key.toString();Vector hisfri
7、ends = new Vector();for (Text val : values) String eachterm = val.toString().split(t);if(eachterm0.equals(someperson)hisfriends.add(eachterm1);job1reduce_value.set(deg1friend);context.write(val, job1reduce_value);else if(eachterm1.equals(someperson)hisfriends.add(eachterm0);job1reduce_value.set(deg1
8、friend);context.write(val, job1reduce_value);for(int i = 0; ihisfriends.size(); i+)for(int j = 0; jhisfriends.size(); j+)if (hisfriends.elementAt(i).compareTo(hisfriends.elementAt(j)0)/job1reduce_key.set(hisfriends.elementAt(j)+t+hisfriends.elementAt(i);/ public static class job2Mapper extends Mappe
9、rprivate Text job2map_key = new Text();private Text job2map_value = new Text(); public void map(Object key, Text value, Context context) throws IOException, InterruptedException String lineterms = value.toString().split(t); if(lineterms.length = 3) job2map_key.set(lineterms0+t+lineterms1); job2map_v
10、alue.set(lineterms2); context.write(job2map_key,job2map_value); public static class job2Reducer extends Reducer private Text job2reducer_key = new Text();private Text job2reducer_value = new Text();public void reduce(Text key, Iterable values, Context context) throws IOException, InterruptedExceptio
11、n Vector relationtags = new Vector();String deg2friendpair = key.toString();for (Text val : values) relationtags.add(val.toString();boolean isadeg1friendpair = false;boolean isadeg2friendpair = false;int surport = 0;for(int i = 0; irelationtags.size(); i+)if(relationtags.elementAt(i).equals(deg1friend)isadeg1friendpair = true;else if(relationtags.elementAt(i).equals(de
