的Hadoop的map-reduce映射编程

Question

import java.io.IOException; 
import org.apache.hadoop.io.LongWritable; 
import org.apache.hadoop.io.Text; 
import org.apache.hadoop.mapred.MapReduceBase; 
import org.apache.hadoop.mapred.Mapper; 
import org.apache.hadoop.mapred.OutputCollector; 
import org.apache.hadoop.mapred.Reporter; 


public class ADDMapper extends MapReduceBase implements Mapper<LongWritable, 
           Text,Text,LongWritable> 
{ @Override 
public void map(LongWritable key, Text value,OutputCollector<Text, LongWritable> output, Reporter r)throws IOException 
    { 
    String s=value.toString(); 
     char[] words=s.toCharArray(); 
        int wno=0; 
        int ino=0; 
     for(int i=0;i<words.length;i++) 
      {  

      String temp=""; 
       for(int j=ino;j<words.length;j++) 
        {       

         if(words[j]!=' ') 
         { temp+=words[j]; 
         } 
         else 
         { 
          wno=j; 
         if(temp!="") 
         {  

          ino=ino + key; //////POINT OF ERROR 

     output.collect(new Text(temp),new LongWritable(ino)); 
         } 

        temp=""; 

         ino=wno+1; 
         break; 
         } 

        } 
     } 
} 

}

我想每个字符串的索引值，通过串排序。
上面的代码既没有给出索引值，也没有对字符串进行混洗。 让 输入文件： 嗨，你好吗 嗨，我是对的。 你的工作情况如何。 你好吗。

输出： 上午50 是7.33 喜0,30,44 如何3,14 。 。

Answer 1

请运行下面的代码，它运行良好，并提供您的预期输出。

提供输入和输出路径在命令行参数。（参数[0]，ARGS [1]）

import java.io.IOException; 
import java.util.*; 
import java.util.Map.Entry; 
import org.apache.hadoop.fs.Path; 
import org.apache.hadoop.io.*; 
import org.apache.hadoop.mapred.*; 


    public class IndexCount { 

     public static class Map extends MapReduceBase implements Mapper<LongWritable, Text, Text, IntWritable> { 
     public void map(LongWritable key, Text value, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException { 

      String str=value.toString(); 
      String[] tokens = str.split(" "); //split into words 
      //create hashmap for unique word 
      HashMap<String,Integer> uniqueString = new HashMap<String,Integer>(); 
      for(int i=0;i<tokens.length;i++){ 
       uniqueString.put(tokens[i],1); 
      }  
      //for sorting create TreeMap from above hash map 
      TreeMap<String, Integer> map = new TreeMap<String,Integer>(uniqueString); 
      for (Entry<String, Integer> entry : map.entrySet()) { 
       int index=0; 
      //find the index of the word 
       index = str.indexOf((String)entry.getKey()); 
       while (index >= 0) { 
         output.collect(new Text((String)entry.getKey()),new IntWritable(index)); 
         index = str.indexOf((String)entry.getKey(), index + 1); 
       } 
      } 
     } 
    } 
     public static class Reduce extends MapReduceBase implements Reducer<Text, IntWritable, Text, IntWritable> { 
     public void reduce(Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException { 

      while (values.hasNext()) { 
       output.collect(key, new IntWritable(values.next().get())); 
      } 

     } 
    } 
     public static void main(String[] args) throws Exception { 
     JobConf conf = new JobConf(WordCount.class); 
     conf.setJobName("indexfinder"); 

     conf.setOutputKeyClass(Text.class); 
     conf.setOutputValueClass(IntWritable.class); 
     conf.setMapperClass(Map.class); 
     conf.setCombinerClass(Reduce.class); 
     conf.setReducerClass(Reduce.class);  
     conf.setInputFormat(TextInputFormat.class); 
     conf.setOutputFormat(TextOutputFormat.class); 

     FileInputFormat.setInputPaths(conf, new Path(args[0])); 
     FileOutputFormat.setOutputPath(conf, new Path(args[1])); 

     JobClient.runJob(conf); 
     } 
    } 

Answer 2

嗨Shivendra我写了下面的映射器的逻辑，这将帮助你找到排序输出每个字符串的索引。 这段代码的输出是用索引对String排序的，然后你可以在这个输出上运行reducer。

String str=value.toString(); 
String[] tokens = str.split(" "); //split into words 
//create hashmap for unique word 
Map<String,Integer> uniqueString = new HashMap<String,Integer>(); 
for(int i=0;i<tokens.length;i++){ 
    uniqueString.put(tokens[i],1); 
}  
//for sorting create TreeMap from above hash map 
Map<String,Integer> map = new TreeMap<String,Integer>(uniqueString); 
for (Map.Entry entry : map.entrySet()) { 
    int index=0; 
//find the index of the word 
    index = str.indexOf((String)entry.getKey()); 
    while (index >= 0) { 
      output.collect(new Text((String)entry.getKey()),new LongWritable(index)); 
      index = str.indexOf((String)entry.getKey(), index + 1); 
    } 
} 

输出这个逻辑： AM：20， 是：7， 是：50， 喜：0， 喜：15， 喜：47， 如何：3， 如何： 30， I：1， I：16， I：18， I：24， I：34， I：48， 是：34， job.:42， ok.:58， right：23， you：11， 你：37， 你：54， 你的：37

它可能会帮助你。

Answer 3

请运行下面的代码，它的给定期望的输出。

import java.io.IOException; 
    import java.util.*; 
    import java.util.Map.Entry; 

    import org.apache.hadoop.fs.Path; 
    import org.apache.hadoop.conf.*; 
    import org.apache.hadoop.io.*; 
    import org.apache.hadoop.mapreduce.*; 
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 
    import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; 
    import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 
    import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; 

    public class Index { 

     public static class Map extends Mapper<LongWritable, Text, Text, IntWritable> { 


     public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { 
      String str=value.toString(); 
       String[] tokens = str.split(" "); //split into words 
       //create hashmap for unique word 
       HashMap<String,Integer> uniqueString = new HashMap<String,Integer>(); 
       for(int i=0;i<tokens.length;i++){ 
        uniqueString.put(tokens[i],1); 
       }  
       //for sorting create TreeMap from above hash map 
       TreeMap<String, Integer> map = new TreeMap<String,Integer>(uniqueString); 
       Configuration conf=context.getConfiguration(); 
       int strIndex = 0; 
       for (Entry<String, Integer> entry : map.entrySet()) { 
        //int index=0; 
        strIndex=conf.getInt("index", 0); 
       //find the index of the word 
        int index = str.indexOf((String)entry.getKey()); 
        while (index >= 0) { 
          index+=strIndex; 
          context.write(new Text((String)entry.getKey()),new IntWritable(index)); 
          index = str.indexOf((String)entry.getKey(), index + 1); 
        } 
       } 
       conf.setInt("index", strIndex+str.length()); 
      } 
     } 

    public static class Reduce extends Reducer<Text, IntWritable, Text, IntWritable> { 

    public void reduce(Text key, Iterable<IntWritable> values, Context context) 
     throws IOException, InterruptedException { 

     for (IntWritable val : values) { 
      context.write(key, new IntWritable(val.get())); 
     } 
    } 
    } 

    public static void main(String[] args) throws Exception { 
    Configuration conf = new Configuration(); 

     conf.setInt("index", 0); 
     Job job = new Job(conf, "index"); 
    job.setOutputKeyClass(Text.class); 
    job.setOutputValueClass(IntWritable.class); 

    job.setMapperClass(Map.class); 
    job.setReducerClass(Reduce.class); 

    job.setInputFormatClass(TextInputFormat.class); 
    job.setOutputFormatClass(TextOutputFormat.class); 

    FileInputFormat.addInputPath(job, new Path("input")); 
    FileOutputFormat.setOutputPath(job, new Path("output")); 

    job.waitForCompletion(true); 
    } 

}