Commit 6415d131 authored by AmourDeMai's avatar AmourDeMai

lastest version

parent 5adb5fb5
This diff is collapsed.
......@@ -87,123 +87,3 @@ public class StringToIntMapWritable implements Writable {
return result + " }";
}
}
/*
* Very simple (and scholastic) implementation of a Writable associative array for String to Int
*
**/
/*
public class StringToIntMapWritable implements Writable {
// TODO: add an internal field that is the real associative array
private HashMap<Text, IntWritable> hashMap;
private Text word;
private IntWritable count;
public StringToIntMapWritable() {
hashMap = new HashMap<Text, IntWritable>();
}
public StringToIntMapWritable(Text text, IntWritable intWritable) {
this();
this.word = text;
this.count = intWritable;
hashMap.put(this.word, this.count);
}
@Override
public void readFields(DataInput in) throws IOException {
// TODO: implement deserialization
// Warning: for efficiency reasons, Hadoop attempts to re-use old instances of
// StringToIntMapWritable when reading new records. Remember to initialize your variables
// inside this function, in order to get rid of old data.
hashMap.clear();
int size = in.readInt();
for (int i = 0; i < size; i++) {
word.readFields(in); // read the size of word from in
count.readFields(in);
hashMap.put(word, count);
}
}
@Override
public void write(DataOutput out) throws IOException {
// TODO: implement serialization
out.writeInt(hashMap.size());
Set<Text> keys = hashMap.keySet();
Iterator iterator = keys.iterator();
while (iterator.hasNext()) {
// write key, write method only write the size of the object bits, here is the size of word
word = (Text) iterator.next();
word.write(out);
// write value
count = hashMap.get(word);
count.write(out);
}
}
public void add(String second, int count) {
hashMap.put(new Text(second), new IntWritable(count));
}
public int getValueWithStringKey(String key) {
if (containsKey(key)) {
return hashMap.get(new Text(key)).get();
} else {
return 0;
}
}
public void addStripe(StringToIntMapWritable stripe) {
Iterator iterator = stripe.getHashMap().keySet().iterator();
while (iterator.hasNext()) {
Text word = (Text)iterator.next();
if (this.hashMap.containsKey(word)) {
this.hashMap.put(word, new IntWritable(this.hashMap.get(word).get() + stripe.hashMap.get(word).get()));
} else {
this.hashMap.put(word, stripe.hashMap.get(word));
}
}
}
public boolean containsKey (String key) {
return hashMap.containsKey(new Text(key));
}
public HashMap<Text, IntWritable> getHashMap() {
return hashMap;
}
public void setHashMap(HashMap<Text, IntWritable> hashMap) {
this.hashMap = hashMap;
}
public String toString() {
String result = " : {";
Iterator iterator = hashMap.keySet().iterator();
while (iterator.hasNext()) {
word = (Text)iterator.next();
count = hashMap.get(iterator.next());
result += word.toString() + " : " + count.toString();
}
return result + " }";
}
}
*/
......@@ -118,65 +118,4 @@ class StripesReducer
}
context.write(key, stripeFinal);
}
}
/*
class StripesMapper
extends Mapper<LongWritable, // TODO: change Object to input key type
Text, // TODO: change Object to input value type
Text, // TODO: change Object to output key type
StringToIntMapWritable> { // TODO: change Object to output value type
@Override
public void map(LongWritable key, // TODO: change Object to input key type
Text value, // TODO: change Object to input value type
Context context)
throws IOException, InterruptedException {
// TODO: implement map method
String line = value.toString();
String[] words = line.split("\\s+");
StringToIntMapWritable stripe = new StringToIntMapWritable();
for (String first : words) {
stripe.getHashMap().clear();
for (String second : words) {
if (first != second) {
if (stripe.containsKey(second)) {
int count = stripe.getValueWithStringKey(second);
count++;
stripe.add(second, count);
} else {
stripe.add(second, (int)1);
}
}
}
context.write(new Text(first), stripe);
}
}
}
class StripesReducer
extends Reducer<Text, // TODO: change Object to input key type
StringToIntMapWritable, // TODO: change Object to input value type
Text, // TODO: change Object to output key type
StringToIntMapWritable> { // TODO: change Object to output value type
@Override
public void reduce(Text key, // TODO: change Object to input key type
Iterable<StringToIntMapWritable> values, // TODO: change Object to input value type
Context context) throws IOException, InterruptedException {
// TODO: implement the reduce method
// the value to send in the end
StringToIntMapWritable stripe = new StringToIntMapWritable();
HashMap<Text, IntWritable> hashMap = stripe.getHashMap();
for (StringToIntMapWritable value : values) {
// get the stripe which is a AssociativeArray, inside in associative array, there's a hashmap
stripe.addStripe(value);
}
context.write(key, stripe);
}
}
*/
\ No newline at end of file
}
\ No newline at end of file
......@@ -88,132 +88,3 @@ public class WordCount extends Configured implements Tool {
ToolRunner.run(new Configuration(), new WordCount(), args);
}
}
/*
import java.io.IOException;
import java.util.Iterator;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
*/
/**
* Word Count example of MapReduce job. Given a plain text in input, this job
* counts how many occurrences of each word there are in that text and writes
* the result on HDFS.
*/
/*
public class WordCount extends Configured implements Tool {
private int numReducers;
private Path inputPath;
private Path outputDir;
@Override
public int run(String[] args) throws Exception {
Configuration conf = this.getConf();
Job job = null; // TODO: define new job instead of null using conf
job = new Job(conf, "Word Count");
// TODO: set job input format\
job.setInputFormatClass(TextInputFormat.class);
// TODO: set map class and the map output key and value classes
job.setMapperClass(WCMapper.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
// TODO: set reduce class and the reduce output key and value classes
job.setReducerClass(WCReducer.class);
// TODO: set job output format
job.setOutputFormatClass(TextOutputFormat.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
// TODO: add the input file as job input (from HDFS) to the variable
// inputPath
FileInputFormat.addInputPath(job, inputPath);
// TODO: set the output path for the job results (to HDFS) to the variable
// outputPath
FileOutputFormat.setOutputPath(job, outputDir);
// TODO: set the number of reducers using variable numberReducers
job.setNumReduceTasks(numReducers);
// TODO: set the jar class
job.setJarByClass(WordCount.class);
return job.waitForCompletion(true) ? 0 : 1; // this will execute the job
}
public WordCount (String[] args) {
if (args.length != 3) {
System.out.println("Usage: WordCount <num_reducers> <input_path> <output_path>");
System.exit(0);
}
this.numReducers = Integer.parseInt(args[0]);
this.inputPath = new Path(args[1]);
this.outputDir = new Path(args[2]);
}
public static void main(String args[]) throws Exception {
int res = ToolRunner.run(new Configuration(), new WordCount(args), args);
System.exit(res);
}
}
class WCMapper extends Mapper<LongWritable, // TODO: change Object to input key type
Text, // TODO: change Object to input value type
Text, // TODO: change Object to output key type
IntWritable> { // TODO: change Object to output value type
private IntWritable ONE = new IntWritable(1);
private Text textValue = new Text();
@Override
protected void map(LongWritable key, // TODO: change Object to input key type
Text value, // TODO: change Object to input value type
Context context) throws IOException, InterruptedException {
// TODO: implement the map method (use context.write to emit results)
String line = value.toString();
String[] words = line.split("\\s+");
for (String word : words) {
textValue.set(word);
context.write(textValue, ONE);
}
}
}
class WCReducer extends Reducer<Text, // TODO: change Object to input key type
IntWritable, // TODO: change Object to input value type
Text, // TODO: change Object to output key type
IntWritable> { // TODO: change Object to output value type
private IntWritable writableSum = new IntWritable();
@Override
protected void reduce(Text key, // TODO: change Object to input key type
Iterable<IntWritable> values, // TODO: change Object to input value type
Context context) throws IOException, InterruptedException {
// TODO: implement the reduce method (use context.write to emit results)
int sum = 0;
for (IntWritable value : values) {
sum += value.get();
}
writableSum.set(sum);
context.write(key, writableSum);
}
}
*/
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment