小编给大家分享一下hadoop-ID的示例分析,相信大部分人都还不怎么了解,因此分享这篇文章给大家参考一下,希望大家阅读完这篇文章后大有收获,下面让我们一起去了解一下吧!
我们开始来分析Hadoop MapReduce的内部的运行机制。用户向Hadoop提交Job(作业),作业在JobTracker对象的控制下执行。Job被分解成为Task(任务),分发到集群中,在TaskTracker的控制下运行。Task包括MapTask和ReduceTask,是MapReduce的Map操作和Reduce操作执行的地方。这中任务分布的方法比较类似于HDFS中NameNode和DataNode的分工,NameNode对应的是JobTracker,DataNode对应的是TaskTracker。JobTracker,TaskTracker和MapReduce的客户端通过RPC通信,具体可以参考HDFS部分的分析。
我们先来分析一些辅助类,首先是和ID有关的类,ID的继承树如下:
* Licensed to the Apache Software Foundation (ASF) under one
package org.apache.hadoop.mapreduce;
import java.io.DataInput;
/**
* A general identifier, which internally stores the id
* as an integer. This is the super class of {@link JobID},
* {@link TaskID} and {@link TaskAttemptID}.
*
* @see JobID
* @see TaskID
* @see TaskAttemptID
*/
public abstract class ID implements WritableComparable<ID> {
protected static final char SEPARATOR = '_';
protected int id;
/** constructs an ID object from the given int */
public ID(int id) {
this.id = id;
}
protected ID() {
}
/** returns the int which represents the identifier */
public int getId() {
return id;
}
@Override
public String toString() {
return String.valueOf(id);
}
@Override
public int hashCode() {
return Integer.valueOf(id).hashCode();
}
@Override
public boolean equals(Object o) {
if (this == o)
return true;
if(o == null)
return false;
if (o.getClass() == this.getClass()) {
ID that = (ID) o;
return this.id == that.id;
}
else
return false;
}
/** Compare IDs by associated numbers */
public int compareTo(ID that) {
return this.id - that.id;
}
public void readFields(DataInput in) throws IOException {
this.id = in.readInt();
}
public void write(DataOutput out) throws IOException {
out.writeInt(id);
}
}
* Licensed to the Apache Software Foundation (ASF) under one
package org.apache.hadoop.mapreduce;
import java.io.DataInput;
/**
* JobID represents the immutable and unique identifier for
* the job. JobID consists of two parts. First part
* represents the jobtracker identifier, so that jobID to jobtracker map
* is defined. For cluster setup this string is the jobtracker
* start time, for local setting, it is "local".
* Second part of the JobID is the job number. <br>
* An example JobID is :
* <code>job_200707121733_0003</code> , which represents the third job
* running at the jobtracker started at <code>200707121733</code>.
* <p>
* Applications should never construct or parse JobID strings, but rather
* use appropriate constructors or {@link #forName(String)} method.
*
* @see TaskID
* @see TaskAttemptID
* @see org.apache.hadoop.mapred.JobTracker#getNewJobId()
* @see org.apache.hadoop.mapred.JobTracker#getStartTime()
*/
public class JobID extends org.apache.hadoop.mapred.ID
implements Comparable<ID> {
protected static final String JOB = "job";
private final Text jtIdentifier;
protected static final NumberFormat idFormat = NumberFormat.getInstance();
static {
idFormat.setGroupingUsed(false);
idFormat.setMinimumIntegerDigits(4);
}
/**
* Constructs a JobID object
* @param jtIdentifier jobTracker identifier
* @param id job number
*/
public JobID(String jtIdentifier, int id) {
super(id);
this.jtIdentifier = new Text(jtIdentifier);
}
public JobID() {
jtIdentifier = new Text();
}
public String getJtIdentifier() {
return jtIdentifier.toString();
}
@Override
public boolean equals(Object o) {
if (!super.equals(o))
return false;
JobID that = (JobID)o;
return this.jtIdentifier.equals(that.jtIdentifier);
}
/**Compare JobIds by first jtIdentifiers, then by job numbers*/
@Override
public int compareTo(ID o) {
JobID that = (JobID)o;
int jtComp = this.jtIdentifier.compareTo(that.jtIdentifier);
if(jtComp == 0) {
return this.id - that.id;
}
else return jtComp;
}
/**
* Add the stuff after the "job" prefix to the given builder. This is useful,
* because the sub-ids use this substring at the start of their string.
* @param builder the builder to append to
* @return the builder that was passed in
*/
public StringBuilder appendTo(StringBuilder builder) {
builder.append(SEPARATOR);
builder.append(jtIdentifier);
builder.append(SEPARATOR);
builder.append(idFormat.format(id));
return builder;
}
@Override
public int hashCode() {
return jtIdentifier.hashCode() + id;
}
@Override
public String toString() {
return appendTo(new StringBuilder(JOB)).toString();
}
@Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
this.jtIdentifier.readFields(in);
}
@Override
public void write(DataOutput out) throws IOException {
super.write(out);
jtIdentifier.write(out);
}
/** Construct a JobId object from given string
* @return constructed JobId object or null if the given String is null
* @throws IllegalArgumentException if the given string is malformed
*/
public static JobID forName(String str) throws IllegalArgumentException {
if(str == null)
return null;
try {
String[] parts = str.split("_");
if(parts.length == 3) {
if(parts[0].equals(JOB)) {
return new org.apache.hadoop.mapred.JobID(parts[1],
Integer.parseInt(parts[2]));
}
}
}catch (Exception ex) {//fall below
}
throw new IllegalArgumentException("JobId string : " + str
+ " is not properly formed");
}
}
以上是“hadoop-ID的示例分析”这篇文章的所有内容,感谢各位的阅读!相信大家都有了一定的了解,希望分享的内容对大家有所帮助,如果还想学习更多知识,欢迎关注亿速云行业资讯频道!
亿速云「云服务器」,即开即用、新一代英特尔至强铂金CPU、三副本存储NVMe SSD云盘,价格低至29元/月。点击查看>>
免责声明:本站发布的内容(图片、视频和文字)以原创、转载和分享为主,文章观点不代表本网站立场,如果涉及侵权请联系站长邮箱:is@yisu.com进行举报,并提供相关证据,一经查实,将立刻删除涉嫌侵权内容。
原文链接:https://my.oschina.net/winHerson/blog/190579