Commit 6a63c81b authored by YUSHIQIAN's avatar YUSHIQIAN
Browse files

implementation

parent 1f1f2ed2
......@@ -38,56 +38,34 @@
<file leaf-file-name="WordCountIMC.java" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/src/main/java/fr/eurecom/dsg/mapreduce/WordCountIMC.java">
<provider selected="true" editor-type-id="text-editor">
<state vertical-scroll-proportion="0.5344828" vertical-offset="0" max-vertical-offset="1425">
<caret line="31" column="48" selection-start-line="31" selection-start-column="48" selection-end-line="31" selection-end-column="48" />
<state vertical-scroll-proportion="0.71005154" vertical-offset="289" max-vertical-offset="1785">
<caret line="56" column="72" selection-start-line="56" selection-start-column="72" selection-end-line="56" selection-end-column="72" />
<folding>
<element signature="imports" expanded="true" />
<element signature="e#1235#1253#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file leaf-file-name="WordCountIMCTest.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/test/java/fr/eurecom/dsg/mapreduce/WordCountIMCTest.java">
<provider selected="true" editor-type-id="text-editor">
<state vertical-scroll-proportion="0.0" vertical-offset="0" max-vertical-offset="495">
<caret line="36" column="5" selection-start-line="36" selection-start-column="5" selection-end-line="36" selection-end-column="5" />
<folding />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="cloud_computing_lab1.iml" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/cloud_computing_lab1.iml">
<provider selected="true" editor-type-id="text-editor">
<state vertical-scroll-proportion="0.0" vertical-offset="0" max-vertical-offset="375">
<caret line="0" column="0" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="pom.xml" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state vertical-scroll-proportion="0.0" vertical-offset="0" max-vertical-offset="1515">
<caret line="19" column="35" selection-start-line="19" selection-start-column="35" selection-end-line="19" selection-end-column="35" />
<folding />
</state>
</provider>
</entry>
</file>
</leaf>
</component>
<component name="IdeDocumentHistory">
<option name="CHANGED_PATHS">
<list>
<option value="$PROJECT_DIR$/src/main/java/fr/eurecom/dsg/mapreduce/WordCount.java" />
<option value="$PROJECT_DIR$/src/main/java/fr/eurecom/dsg/mapreduce/WordCountIMC.java" />
<option value="$PROJECT_DIR$/src/test/java/fr/eurecom/dsg/mapreduce/WordCountIMCTest.java" />
<option value="$PROJECT_DIR$/src/main/java/fr/eurecom/dsg/mapreduce/WordCountIMC.java" />
</list>
</option>
</component>
<component name="MavenImportPreferences">
<option name="importingSettings">
<MavenImportingSettings>
<option name="importAutomatically" value="true" />
</MavenImportingSettings>
</option>
</component>
<component name="ProjectFrameBounds">
<option name="x" value="-1" />
<option name="y" value="27" />
......@@ -194,7 +172,7 @@
<property name="MemberChooser.sorted" value="false" />
<property name="MemberChooser.showClasses" value="true" />
<property name="MemberChooser.copyJavadoc" value="false" />
<property name="options.lastSelected" value="preferences.pathVariables" />
<property name="options.lastSelected" value="reference.settings.project.maven.importing" />
<property name="options.splitter.main.proportions" value="0.3" />
<property name="options.splitter.details.proportions" value="0.2" />
</component>
......@@ -375,15 +353,16 @@
<window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Palette" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" weight="0.32998884" sideWeight="0.5" order="-1" side_tool="true" content_ui="tabs" />
<window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" weight="0.23411371" sideWeight="0.5" order="-1" side_tool="true" content_ui="tabs" />
<window_info id="Favorites" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" weight="0.33" sideWeight="0.5" order="-1" side_tool="true" content_ui="tabs" />
<window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Messages" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" weight="0.10479376" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" weight="0.33" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" />
<window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Maven Projects" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Commander" active="false" anchor="right" auto_hide="false" internal_type="SLIDING" type="SLIDING" visible="false" weight="0.4" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
<window_info id="Application Servers" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" weight="0.24973376" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
<window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" weight="0.20074548" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
<window_info id="Cvs" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" weight="0.25" sideWeight="0.5" order="4" side_tool="false" content_ui="tabs" />
<window_info id="Message" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" weight="0.33" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
<window_info id="Find" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" weight="0.33" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
......@@ -410,9 +389,15 @@
</option>
</component>
<component name="XDebuggerManager">
<breakpoint-manager />
<breakpoint-manager>
<option name="time" value="3" />
</breakpoint-manager>
<watches-manager />
</component>
<component name="antWorkspaceConfiguration">
<option name="IS_AUTOSCROLL_TO_SOURCE" value="false" />
<option name="FILTER_TARGETS" value="false" />
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/src/main/main.iml">
<provider selected="true" editor-type-id="text-editor">
......@@ -430,36 +415,39 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/cloud_computing_lab1.iml">
<entry file="file://$PROJECT_DIR$/src/test/java/fr/eurecom/dsg/mapreduce/WordCountIMCTest.java">
<provider selected="true" editor-type-id="text-editor">
<state vertical-scroll-proportion="0.0" vertical-offset="0" max-vertical-offset="574">
<caret line="0" column="0" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
<state vertical-scroll-proportion="0.7055749" vertical-offset="0" max-vertical-offset="574">
<caret line="36" column="5" selection-start-line="36" selection-start-column="5" selection-end-line="36" selection-end-column="5" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/pom.xml">
<entry file="file://$PROJECT_DIR$/cloud_computing_lab1.iml">
<provider selected="true" editor-type-id="text-editor">
<state vertical-scroll-proportion="0.0" vertical-offset="0" max-vertical-offset="1515">
<caret line="19" column="35" selection-start-line="19" selection-start-column="35" selection-end-line="19" selection-end-column="35" />
<state vertical-scroll-proportion="0.0" vertical-offset="0" max-vertical-offset="574">
<caret line="0" column="0" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/test/java/fr/eurecom/dsg/mapreduce/WordCountIMCTest.java">
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state vertical-scroll-proportion="0.0" vertical-offset="0" max-vertical-offset="495">
<caret line="36" column="5" selection-start-line="36" selection-start-column="5" selection-end-line="36" selection-end-column="5" />
<state vertical-scroll-proportion="1.097561" vertical-offset="0" max-vertical-offset="1515">
<caret line="42" column="25" selection-start-line="42" selection-start-column="25" selection-end-line="42" selection-end-column="25" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/fr/eurecom/dsg/mapreduce/WordCountIMC.java">
<provider selected="true" editor-type-id="text-editor">
<state vertical-scroll-proportion="0.5344828" vertical-offset="0" max-vertical-offset="1425">
<caret line="31" column="48" selection-start-line="31" selection-start-column="48" selection-end-line="31" selection-end-column="48" />
<state vertical-scroll-proportion="0.71005154" vertical-offset="289" max-vertical-offset="1785">
<caret line="56" column="72" selection-start-line="56" selection-start-column="72" selection-end-line="56" selection-end-column="72" />
<folding>
<element signature="imports" expanded="true" />
<element signature="e#1235#1253#0" expanded="true" />
</folding>
</state>
</provider>
......
package fr.eurecom.dsg.mapreduce;
import java.io.IOException;
import java.util.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
......@@ -18,6 +19,7 @@ import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
public class WordCountIMC extends Configured implements Tool {
static class WCMapper extends Mapper<LongWritable, Text, Text, IntWritable> {
......@@ -28,11 +30,32 @@ public class WordCountIMC extends Configured implements Tool {
@Override
protected void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
HashMap<String , Integer> map = new HashMap<String , Integer>();
String line = value.toString();
String[] words = line.split("\\s+");
for(String word : words) {
textValue.set(word);
context.write(textValue, ONE);}
if (map.containsKey(word)) {
map.put(word, map.get(word) + 1);
}
else {
map.put(word, 1);
}
//context.write(textValue, ONE);
}
Iterator it = map.entrySet().iterator();
while (it.hasNext()) {
Map.Entry pair = (Map.Entry)it.next();
context.write(pair.getKey(), pair.getValue());
it.remove(); // avoids a ConcurrentModificationException
}
}
}
......@@ -43,6 +66,7 @@ public class WordCountIMC extends Configured implements Tool {
@Override
protected void reduce(Text key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int sum = 0;
for (IntWritable value : values)
sum += value.get();
......@@ -57,7 +81,7 @@ public class WordCountIMC extends Configured implements Tool {
Configuration conf = this.getConf();
Job job = new Job(conf,"Word Count");
Job job = new Job(conf,"Word Count IMC");
job.setInputFormatClass(TextInputFormat.class);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment