hadoop编译

我是hadoop的新手。我从网上hadoop编译

import java.io.IOException; 
import java.util.*; 

import org.apache.hadoop.fs.Path; 
import org.apache.hadoop.conf.*; 
import org.apache.hadoop.io.*; 
import org.apache.hadoop.mapred.*; 
import org.apache.hadoop.util.*; 

public class Gender { 

    private static String genderCheck = "female"; 

    public static class Map extends MapReduceBase implements Mapper { 
     private final static IntWritable one = new IntWritable(1); 
     private Text locText = new Text(); 

     public void map(LongWritable key, Text value, OutputCollector output, Reporter reporter) throws IOException { 
      String line = value.toString(); 
      String location = line.split(",")[14] + "," + line.split(",")[15]; 
      long male = 0L; 
      long female = 0L; 
      if (line.split(",")[17].matches("\d+") && line.split(",")[18].matches("\d+")) { 
       male = Long.parseLong(line.split(",")[17]); 
       female = Long.parseLong(line.split(",")[18]); 
      } 
      long diff = male - female; 
      locText.set(location); 
      if (Gender.genderCheck.toLowerCase().equals("female") && diff < 0) { 
       output.collect(locText, new LongWritable(diff * -1L)); 
      } 
      else if (Gender.genderCheck.toLowerCase().equals("male") && diff 
> 0) { 
       output.collect(locText, new LongWritable(diff)); 
      } 
     } } 

    public static void main(String[] args) throws Exception { 
     JobConf conf = new JobConf(Gender.class); 
     conf.setJobName("gender"); 
     conf.setOutputKeyClass(Text.class); 
     conf.setOutputValueClass(LongWritable.class); 
     conf.setMapperClass(Map.class); 

     if (args.length != 3) { 
      System.out.println("Usage:"); 
      System.out.println("[male/female] /path/to/2kh/files /path/to/output"); 
      System.exit(1); 
     } 

     if (!args[0].equalsIgnoreCase("male") && !args[0].equalsIgnoreCase("female")) { 
      System.out.println("first argument must be male or female"); 
      System.exit(1); 
     } 
     Gender.genderCheck = args[0]; 

     conf.setInputFormat(TextInputFormat.class); 
     conf.setOutputFormat(TextOutputFormat.class); 
     FileInputFormat.setInputPaths(conf, new Path(args[1])); 
     FileOutputFormat.setOutputPath(conf, new Path(args[2])); 
     JobClient.runJob(conf); } 

}

此代码时，我使用“的javac -cp /usr/local/hadoop/hadoop-core-1.0.3.jar Gender.java”

得到以下编译这段代码错误：

如何正确编译它？

来源

2013-04-08 Robin

更改类Maper类声明如下：

public static class Map extends MapReduceBase implements Mapper<LongWritable,Text,Text, LongWritable>

如果不指定任何特定的类名，你就需要有地图的功能如下：

@Override 
public void map(Object arg0, Object arg1, OutputCollector arg2, Reporter arg3) throws IOException { 
    // TODO Auto-generated method stub 
}

现在，这里的特定类型表示预期的输入键值对类型和来自映射器的输出键值类型。

在你的情况下，输入键值对是LongWritable-Text。

而且，从output.collect方法调用中猜测，您的映射器输出键值对是Text-LongWritable。

因此，您的地图类应执行Mapper<LongWritable,Text,Text, LongWritable>。

有在代码中多了一个错误 -

使用"\d+"将无法编译为\d没有意义，反斜线后，预计一个特殊的转义序列，所以我想对你下面应该工作： line.split(",")[17].matches("\\d+")

来源

2013-04-08 07:07:41 Amar

更改地图类，如下所示：

public static class Map extends MapReduceBase implements Mapper <Input key, Input value, Output Key , Output Value>

在你的情况下，输入键是LongWritable，输入值是Text，输出密钥是Text，输出值是LongWritable

public static class Map extends MapReduceBase implements Mapper <LongWritable, Text, Text,LongWritable>

来源

2017-09-12 07:22:49

回答

相关问题