这期内容当中小编将会给大家带来有关hadoop中怎么将文件上传到指定datanode,文章内容丰富且以专业的角度为大家分析和叙述,阅读完这篇文章希望大家可以有所收获。
package hgs.dfsclient.test;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsCreateModes;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DFSOutputStream;
import org.apache.hadoop.hdfs.protocol.*;
import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.EnumSet;
import java.util.List;
import java.util.Random;
public class MainTest {
public static void main(String[] args) throws IOException, URISyntaxException {
Configuration conf = new Configuration();
conf.addResource(new File("D://hdfsconf/core-site.xml").toURI().toURL());
conf.addResource(new File("D://hdfsconf/hdfs-site.xml").toURI().toURL());
String url = conf.get("fs.defaultFS");
System.out.println(url);
Path path = new Path("/user/test.txt");
DFSClient client = new DFSClient(new URI("hdfs://192.168.0.191:9000/"),conf);
//favor datanode
InetSocketAddress [] favor = {new InetSocketAddress("192.168.0.191", 50012)};
createAndFillFile(client,conf,favor,"/user/test.txt4");
listBlockLocation(client, conf, "/user/test.txt4",true );
client.close();
}
//将数据传输到指定的datanode
public static void createAndFillFile(DFSClient client,Configuration conf,
InetSocketAddress [] favor,String src) throws IOException {
DFSOutputStream dfsOutputStream = client.create(src, FsCreateModes.applyUMask(
FsPermission.getFileDefault(), FsPermission.getUMask(conf)), EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
true, (short) 1, 32 * 1024 * 1024,
null, 8192, null, favor, null);
byte[] b = new byte[1024*3*3];
for(int i = 0 ;i<100000;i++){
fileByte(b);
dfsOutputStream.write(b);
}
dfsOutputStream.close();
}
static Random rand = new Random(System.nanoTime());
public static void fileByte(byte[] b){
if(b != null ){
rand.nextBytes(b);
}
}
//获取文件的block位置
public static void listBlockLocation(DFSClient client,Configuration conf,String src,boolean needLocation) throws IOException {
DirectoryListing directoryListing = client.listPaths(src, HdfsFileStatus.EMPTY_NAME,
needLocation);
HdfsFileStatus[] fileStatuses = directoryListing.getPartialListing();
for(int i=0 ; i <fileStatuses.length ; i++){
HdfsLocatedFileStatus hl = (HdfsLocatedFileStatus)fileStatuses[i];
LocatedBlocks locatedBlocks = hl.getLocatedBlocks();
List<LocatedBlock> locatedBlocks1 = locatedBlocks.getLocatedBlocks();
for(LocatedBlock lb : locatedBlocks1 ){
DatanodeInfo[] locations = lb.getLocations();
for(DatanodeInfo li : locations){
System.out.println("addr:"+li.getIpAddr()+"--port:"+li);
}
}
}
}
}
上述就是小编为大家分享的hadoop中怎么将文件上传到指定datanode了,如果刚好有类似的疑惑,不妨参照上述分析进行理解。如果想知道更多相关知识,欢迎关注亿速云行业资讯频道。
亿速云「云服务器」,即开即用、新一代英特尔至强铂金CPU、三副本存储NVMe SSD云盘,价格低至29元/月。点击查看>>
免责声明:本站发布的内容(图片、视频和文字)以原创、转载和分享为主,文章观点不代表本网站立场,如果涉及侵权请联系站长邮箱:is@yisu.com进行举报,并提供相关证据,一经查实,将立刻删除涉嫌侵权内容。
原文链接:http://blog.itpub.net/31506529/viewspace-2698379/