Commit 6a63c81b authored by YUSHIQIAN's avatar YUSHIQIAN

implementation

parent 1f1f2ed2
This diff is collapsed.
package fr.eurecom.dsg.mapreduce;
import java.io.IOException;
import java.util.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
......@@ -18,6 +19,7 @@ import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
public class WordCountIMC extends Configured implements Tool {
static class WCMapper extends Mapper<LongWritable, Text, Text, IntWritable> {
......@@ -28,11 +30,32 @@ public class WordCountIMC extends Configured implements Tool {
@Override
protected void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
HashMap<String , Integer> map = new HashMap<String , Integer>();
String line = value.toString();
String[] words = line.split("\\s+");
for(String word : words) {
textValue.set(word);
context.write(textValue, ONE);}
if (map.containsKey(word)) {
map.put(word, map.get(word) + 1);
}
else {
map.put(word, 1);
}
//context.write(textValue, ONE);
}
Iterator it = map.entrySet().iterator();
while (it.hasNext()) {
Map.Entry pair = (Map.Entry)it.next();
context.write(pair.getKey(), pair.getValue());
it.remove(); // avoids a ConcurrentModificationException
}
}
}
......@@ -43,6 +66,7 @@ public class WordCountIMC extends Configured implements Tool {
@Override
protected void reduce(Text key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int sum = 0;
for (IntWritable value : values)
sum += value.get();
......@@ -57,7 +81,7 @@ public class WordCountIMC extends Configured implements Tool {
Configuration conf = this.getConf();
Job job = new Job(conf,"Word Count");
Job job = new Job(conf,"Word Count IMC");
job.setInputFormatClass(TextInputFormat.class);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment