本篇文章为大家展示了如何在Hadoop项目中使用Spring，内容简明扼要并且容易理解，绝对能使你眼前一亮，通过这篇文章的详细介绍希望你能有所收获。
计算
安全
数据库
网络和加速
企业服务
<dependencies>
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-hadoop</artifactId>
<version>2.5.0.RELEASE</version>
</dependency>
</dependencies>新建资源文件beans.xml
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:hdp="http://www.springframework.org/schema/hadoop"
xsi:schemaLocation="
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/hadoop
http://www.springframework.org/schema/hadoop/spring-hadoop.xsd">
<hdp:configuration id="hadoopConfiguration">
fs.defaultFS=hdfs://hadoop01:9000
hadoop.tmp.dir=/tmp/hadoop
electric=sea
</hdp:configuration>
<hdp:file-system id="fileSystem"
configuration-ref="hadoopConfiguration"
user="root"
/>
</beans>测试文件
package com.bennyrhys.hadoop.spring;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import java.io.IOException;
/**
* @Author bennyrhys
* @Date 1/21/21 2:35 PM
*/
public class SpringHadoopHDFSApp {
private ApplicationContext ctx;
// apache hadoop
private FileSystem fileSystem;
/**
* 创建HDFS文件夹
*/
@Test
public void testMkdirs() throws Exception {
fileSystem.mkdirs(new Path("/springhdfs"));
}
/**
* 查看HDFS文件
*/
@Test
public void testText() throws Exception {
FSDataInputStream in = fileSystem.open(new Path("/springhdfs/hello.txt"));
IOUtils.copyBytes(in, System.out, 1024);
in.close();
}
@Before
public void setUp() {
ctx = new ClassPathXmlApplicationContext("beans.xml");
fileSystem = (FileSystem) ctx.getBean("fileSystem");
}
@After
public void tearDown() throws Exception {
ctx = null;
fileSystem.close();
}
}提取变量
使用xml中的头文件替换bean，使其允许使用上下文
${}导入变量
新建配置文件application.properties
spring.hadoop.fsUri=hdfs://hadoop01:9000获取context上下文引入变量
beans.xml
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:hdp="http://www.springframework.org/schema/hadoop"
xmlns:context="http://www.springframework.org/schema/context"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd
http://www.springframework.org/schema/hadoop http://www.springframework.org/schema/hadoop/spring-hadoop.xsd">
<hdp:configuration id="hadoopConfiguration">
fs.defaultFS=${spring.hadoop.fsUri}
hadoop.tmp.dir=/tmp/hadoop
electric=sea
</hdp:configuration>
<context:property-placeholder location="application.properties"/>
<hdp:file-system id="fileSystem"
configuration-ref="hadoopConfiguration"
user="root"
/>
</beans>pom.xml
<!-- 添加spring boot的依赖操作hadoop -->
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-hadoop-boot</artifactId>
<version>2.5.0.RELEASE</version>
</dependency>SpringBootHDFSApp
package com.bennyrhys.hadoop.spring;
import org.apache.hadoop.fs.FileStatus;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.data.hadoop.fs.FsShell;
/**
* @Author bennyrhys
* @Date 1/21/21 11:33 PM
*/
@SpringBootApplication
public class SpringBootHDFSApp implements CommandLineRunner {
@Autowired
FsShell fsShell; //引入spring的
@Override
public void run(String... strings) throws Exception {
for (FileStatus fileStatus : fsShell.lsr("/springhdfs")) {
System.out.println("> " + fileStatus.getPath());
}
}
/**
* > hdfs://hadoop01:9000/springhdfs
* > hdfs://hadoop01:9000/springhdfs/hello.txt
* @param args
*/
public static void main(String[] args) {
SpringApplication.run(SpringBootHDFSApp.class, args);
}
}上述内容就是如何在Hadoop项目中使用Spring，你们学到知识或技能了吗？如果还想学到更多技能或者丰富自己的知识储备，欢迎关注亿速云行业资讯频道。
亿速云「云服务器」，即开即用、新一代英特尔至强铂金CPU、三副本存储NVMe SSD云盘，价格低至29元/月。点击查看>>
免责声明：本站发布的内容（图片、视频和文字）以原创、转载和分享为主，文章观点不代表本网站立场，如果涉及侵权请联系站长邮箱：is@yisu.com进行举报，并提供相关证据，一经查实，将立刻删除涉嫌侵权内容。
