Question

我正在尝试在Oozie上运行MapReduce作业，但它失败并被杀死。该错误也未显示在oozie控制台中，它仅显示此错误消息：“Map / Reduce failed，error message []”。另外我在哪里可以找到日志，它是否包含确切的错误。我是新手，不知道出了什么问题。请告诉我以下代码有什么问题。我现在忍受这两天了。

这是我的MapReduce计划。

package com.hadoop.mapreduce;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class AnalysePatientRecords {

    public static class SampleMap extends Mapper<LongWritable, Text, Text, Text> {

        public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

            String line = value.toString();
            String[] lineElements = line.split("\\|",-1);
            Text state = new Text(lineElements[11]);
            Text patientId = new Text(lineElements[0]);
            context.write(state, patientId);

        }

    }

    public static class SampleReduce extends Reducer<Text, Text, Text, IntWritable> {

        @SuppressWarnings("unused")
        public void reduce(Text value, Iterable<Text> values, Context context)
                throws IOException, InterruptedException {

            int count = 0;
            for (Text val : values) {
                count = count + 1;
            }
            context.write(value, new IntWritable(count));

        }

    }

    public static void main(String[] args) throws Exception {

        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf, "patient analysis");
        job.setJarByClass(AnalysePatientRecords.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);
        job.setMapperClass(SampleMap.class);
        job.setReducerClass(SampleReduce.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);
        FileInputFormat.addInputPath(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));
        System.exit(job.waitForCompletion(true) ? 0 : 1);

    }

}

job.properties

nameNode=hdfs://localhost:54310
jobTracker=localhost:8032
queueName=default
examplesRoot=MapReduce

oozie.libpath=${nameNode}/user/${user.name}/share/lib
oozie.use.system.libpath=true

oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/map-reduce/workflow.xml
outputDir=map-reduce

workflow.xml

<workflow-app xmlns="uri:oozie:workflow:0.2" name="map-reduce-wf">
    <start to="mr-node"/>
    <action name="mr-node">
        <map-reduce>
            <job-tracker>${jobTracker}</job-tracker>
            <name-node>${nameNode}</name-node>
            <prepare>
                <delete path="${nameNode}/user/${wf:user()}/${examplesRoot}/output-data/${outputDir}"/>
            </prepare>
            <configuration>
        <property>
                    <name>mapred.mapper.new-api</name>
                    <value>true</value>
                </property>
                <property>
                    <name>mapred.reducer.new-api</name>
                    <value>true</value>
                </property>
                <property>
                    <name>mapred.job.queue.name</name>
                    <value>${queueName}</value>
                </property>
                <property>
                    <name>mapreduce.map.class</name>
                    <value>AnalysePatientRecords$SampleMap</value>
                </property>
                <property>
                    <name>mapreduce.reduce.class</name>
                    <value>AnalysePatientRecords$SampleReduce</value>
                </property>
        <property>
                <name>mapred.mapoutput.key.class</name>
                <value>org.apache.hadoop.io.Text</value>
                </property>
                <property>
                        <name>mapred.mapoutput.value.class</name>
                        <value>org.apache.hadoop.io.Text</value>
                </property>
                <property>
                        <name>mapred.output.key.class</name>
                        <value>org.apache.hadoop.io.Text</value>
                </property>
                <property>
                        <name>mapred.output.value.class</name>
                        <value>org.apache.hadoop.io.IntWritable</value>
                </property>
                <property>
                    <name>mapred.map.tasks</name>
                    <value>1</value>
                </property>
                <property>
                    <name>mapred.input.dir</name>
                    <value>/user/${wf:user()}/${examplesRoot}/input-data/text</value>
                </property>
                <property>
                    <name>mapred.output.dir</name>
                    <value>/user/${wf:user()}/${examplesRoot}/output-data/${outputDir}</value>
                </property>
            </configuration>
        </map-reduce>
        <ok to="end"/>
        <error to="fail"/>
    </action>
    <kill name="fail">
        <message>Map/Reduce failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
    </kill>
    <end name="end"/>
</workflow-app>

我还创建了上面提到的MapReduce程序的jar并将其放在lib文件夹中。我不知道有什么不对，请帮帮我。请...

Answer 1

结帐Oozie Hive Action Logging。正如文档中所述 - “Hive操作日志被重定向到运行Hive的Oozie Launcher map-reduce作业任务STDOUT / STDERR”。 @YoungHobbit还澄清了他们在评论中检查oozie启动器工作日志的信息。

您可以从Oozie Web控制台访问此日志。从Oozie Web控制台，使用“控制台URL”链接弹出的Hive操作，可以通过Hadoop作业跟踪器Web控制台导航到Oozie Launcher map-reduce作业任务日志。

在Oozie中运行MapReduce作业

1 个答案: