mapreduce作业单元测试
用官方提供的MRunit包进行对MapReducer作业进行测试 http://mrunit.apache.org/ 先要下载对应的测试包，分为1.0和2.0的hadoop版本 来自： https://cwiki.apache.org/confluence/display/MRUNIT/MRUnit+Tutorial Mapper Java代码 importorg.apache.hadoop.io.IntWritable; importorg.apache.hadoop.io.LongWritable; importorg.apache.hadoop.io.Text; importorg.apache.hadoop.mapreduce.Mapper; publicclassSMSCDRMapperextendsMapper<LongWritable,Text,Text,IntWritable> { privateTextstatus=newText(); privatefinalstaticIntWritableaddOne=newIntWritable(1); @Override protectedvoidmap(LongWritablekey,Textvalue,Contextcontext)throwsjava.io.IOException,InterruptedException { //655209;1;796764372490213;804422938115889;6istheSamplerecordformat String[]line=value.toString().split(";"); //IfrecordisofSMSCDR if(Integer.parseInt(line[1])==1) { status.set(line[4]); context.write(status,addOne); } } } import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Mapper; public class SMSCDRMapper extends Mapper<LongWritable, Text, Text, IntWritable> { private Text status = new Text(); private final static IntWritable addOne = new IntWritable(1); @Override protected void map(LongWritable key, Text value, Context context) throws java.io.IOException, InterruptedException { //655209;1;796764372490213;804422938115889;6 is the Sample record format String[] line = value.toString().split(";"); // If record is of SMS CDR if (Integer.parseInt(line[1]) == 1) { status.set(line[4]); context.write(status, addOne); } } } Reducer: Java代码 importjava.util.List; importorg.apache.hadoop.io.IntWritable; importorg.apache.hadoop.io.LongWritable; importorg.apache.hadoop.io.Text; importorg.apache.hadoop.mrunit.mapreduce.MapDriver; importorg.apache.hadoop.mrunit.mapreduce.MapReduceDriver; importorg.apache.hadoop.mrunit.mapreduce.ReduceDriver; importorg.junit.Before; importorg.junit.Test; importflex.messaging.io.ArrayList; publicclassSMSCDRMapperReducerTest { MapDriver<LongWritable,Text,Text,IntWritable>mapDriver; ReduceDriver<Text,IntWritable,Text,IntWritable>reduceDriver; MapReduceDriver<LongWritable,Text,Text,IntWritable,Text,IntWritable>mapReduceDriver; @Before publicvoidsetup() { SMSCDRMappermapper=newSMSCDRMapper(); SMSCDRReducerreducer=newSMSCDRReducer(); mapDriver=MapDriver.newMapDriver(mapper); reduceDriver=ReduceDriver.newReduceDriver(reducer); mapReduceDriver=MapReduceDriver.newMapReduceDriver(mapper,reducer); } @Test publicvoidtestMapper() { mapDriver.withInput(newLongWritable(),newText( "655209;1;796764372490213;804422938115889;6")); mapDriver.withOutput(newText("6"),newIntWritable(1)); mapDriver.runTest(); } @Test publicvoidtestReducer() { List<IntWritable>values=newArrayList(); values.add(newIntWritable(1)); values.add(newIntWritable(1)); reduceDriver.withInput(newText("6"),values); reduceDriver.withOutput(newText("6"),newIntWritable(2)); reduceDriver.runTest(); } } import java.util.List; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mrunit.mapreduce.MapDriver; import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver; import org.apache.hadoop.mrunit.mapreduce.ReduceDriver; import org.junit.Before; import org.junit.Test; import flex.messaging.io.ArrayList; public class SMSCDRMapperReducerTest { MapDriver<LongWritable, Text, Text, IntWritable> mapDriver; ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver; MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable> mapReduceDriver; @Before public void setup() { SMSCDRMapper mapper = new SMSCDRMapper(); SMSCDRReducer reducer = new SMSCDRReducer(); mapDriver = MapDriver.newMapDriver(mapper); reduceDriver = ReduceDriver.newReduceDriver(reducer); mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer); } @Test public void testMapper() { mapDriver.withInput(new LongWritable(), new Text( "655209;1;796764372490213;804422938115889;6")); mapDriver.withOutput(new Text("6"), new IntWritable(1)); mapDriver.runTest(); } @Test public void testReducer() { List<IntWritable> values = new ArrayList(); values.add(new IntWritable(1)); values.add(new IntWritable(1)); reduceDriver.withInput(new Text("6"), values); reduceDriver.withOutput(new Text("6"), new IntWritable(2)); reduceDriver.runTest(); } } Test测试类 Java代码 importjava.util.List; importorg.apache.hadoop.io.IntWritable; importorg.apache.hadoop.io.LongWritable; importorg.apache.hadoop.io.Text; importorg.apache.hadoop.mrunit.mapreduce.MapDriver; importorg.apache.hadoop.mrunit.mapreduce.MapReduceDriver; importorg.apache.hadoop.mrunit.mapreduce.ReduceDriver; importorg.junit.Before; importorg.junit.Test; importflex.messaging.io.ArrayList; publicclassSMSCDRMapperReducerTest { MapDriver<LongWritable,Text,Text,IntWritable>mapDriver; ReduceDriver<Text,IntWritable,Text,IntWritable>reduceDriver; MapReduceDriver<LongWritable,Text,Text,IntWritable,Text,IntWritable>mapReduceDriver; @Before publicvoidsetup() { SMSCDRMappermapper=newSMSCDRMapper(); SMSCDRReducerreducer=newSMSCDRReducer(); mapDriver=MapDriver.newMapDriver(mapper); reduceDriver=ReduceDriver.newReduceDriver(reducer); mapReduceDriver=MapReduceDriver.newMapReduceDriver(mapper,reducer); } @Test publicvoidtestMapper() { mapDriver.withInput(newLongWritable(),newText( "655209;1;796764372490213;804422938115889;6")); mapDriver.withOutput(newText("6"),newIntWritable(1)); mapDriver.runTest(); } @Test publicvoidtestReducer() { List<IntWritable>values=newArrayList(); values.add(newIntWritable(1)); values.add(newIntWritable(1)); reduceDriver.withInput(newText("6"),values); reduceDriver.withOutput(newText("6"),newIntWritable(2)); reduceDriver.runTest(); } } import java.util.List; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mrunit.mapreduce.MapDriver; import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver; import org.apache.hadoop.mrunit.mapreduce.ReduceDriver; import org.junit.Before; import org.junit.Test; import flex.messaging.io.ArrayList; public class SMSCDRMapperReducerTest { MapDriver<LongWritable, Text, Text, IntWritable> mapDriver; ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver; MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable> mapReduceDriver; @Before public void setup() { SMSCDRMapper mapper = new SMSCDRMapper(); SMSCDRReducer reducer = new SMSCDRReducer(); mapDriver = MapDriver.newMapDriver(mapper); reduceDriver = ReduceDriver.newReduceDriver(reducer); mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer); } @Test public void testMapper() { mapDriver.withInput(new LongWritable(), new Text( "655209;1;796764372490213;804422938115889;6")); mapDriver.withOutput(new Text("6"), new IntWritable(1)); mapDriver.runTest(); } @Test public void testReducer() { List<IntWritable> values = new ArrayList(); values.add(new IntWritable(1)); values.add(new IntWritable(1)); reduceDriver.withInput(new Text("6"), values); reduceDriver.withOutput(new Text("6"), new IntWritable(2)); reduceDriver.runTest(); } } 转自http://piaoling.iteye.com/blog/1672683 本文转自茄子_2008博客园博客，原文链接：http://www.cnblogs.com/xd502djj/archive/2013/04/09/3011240.html，如需转载请自行联系原作者。
