Spark SQL怎么用

这篇文章主要介绍“Spark SQL怎么用”，在日常操作中，相信很多人在Spark SQL怎么用问题上存在疑惑，小编查阅了各式资料，整理出简单好用的操作方法，希望对大家解答”Spark SQL怎么用”的疑惑有所帮助！接下来，请跟着小编一起来学习吧！

pom.xml
<dependencies>

1.     <dependency>
   

2.         <groupId>org.apache.spark</groupId>
   

3.         <artifactId>spark-core_2.10</artifactId>
   

4.         <version>2.1.0</version>
   

5.     </dependency>
   

6.     <dependency>
   

7.         <groupId>org.apache.spark</groupId>
   

8.         <artifactId>spark-sql_2.10</artifactId>
   

9.         <version>2.1.0</version>
   

10.     </dependency>
    

11. </dependencies>

Java:

1. import java.io.Serializable;
   

2. import java.util.Arrays;
   

3. 
   

4. import org.apache.spark.SparkConf;
   

5. import org.apache.spark.api.java.JavaRDD;
   

6. import org.apache.spark.api.java.JavaSparkContext;
   

7. import org.apache.spark.sql.Dataset;
   

8. import org.apache.spark.sql.Row;
   

9. import org.apache.spark.sql.SQLContext;
   

10. import org.apache.spark.sql.SparkSession;
    

11. 
    

12. public class SparkSqlTest {
    

13.     public static class Person implements Serializable {
    

14.         private static final long serialVersionUID = -6259413972682177507L;
    

15.         private String name;
    

16.         private int age;
    

17.         
    

18.         public Person(String name, int age) {
    

19.             this.name = name;
    

20.             this.age = age;
    

21.         }
    

22.         public String toString() {
    

23.             return name + ": " + age;
    

24.         }
    

25.         public String getName() {
    

26.             return name;
    

27.         }
    

28.         public void setName(String name) {
    

29.             this.name = name;
    

30.         }
    

31.         public int getAge() {
    

32.             return age;
    

33.         }
    

34.         public void setAge(int age) {
    

35.             this.age = age;
    

36.         }
    

37.     }
    

38.     
    

39.     public static void main(String[] args) {
    

40.         SparkConf conf = new SparkConf().setAppName("Test").setMaster("local");
    

41.         JavaSparkContext sc = new JavaSparkContext(conf);
    

42.         
    

43.         SparkSession spark = SparkSession.builder().appName("Test").getOrCreate();
    

44.         JavaRDD<String> input = sc.parallelize(Arrays.asList("abc,1", "test,2"));
    

45.         JavaRDD<Person> persons = input.map(s -> s.split(",")).map(s -> new Person(s[0], Integer.parseInt(s[1])));
    

46.         

47.         //[abc: 1, test: 2]

48.         System.out.println(persons.collect());
    

49.         
    

50.         Dataset<Row> df = spark.createDataFrame(persons, Person.class);
    

51.         
    

52.         /*
    

53.         +---+----+
    

54.         |age|name|
    

55.         +---+----+
    

56.         |  1| abc|
    

57.         |  2|test|
    

58.         +---+----+
    

59.          */
    

60.         df.show();
    

61.         
    

62.         /*
    

63.          root
    

64.            |-- age: integer (nullable = false)
    

65.            |-- name: string (nullable = true)
    

66.          */
    

67.         df.printSchema();
    

68. 
    

69.         SQLContext sql = new SQLContext(spark);
    

70.         sql.registerDataFrameAsTable(df, "person");
    

71.         
    

72.         /*
    

73.         +---+----+
    

74.         |age|name|
    

75.         +---+----+
    

76.         |  2|test|
    

77.         +---+----+
    

78.          */
    

79.         sql.sql("SELECT * FROM person WHERE age>1").show();
    

80.         
    

81.         sc.close();
    

82.     }
    

83. 
    

84. }

到此，关于“Spark SQL怎么用”的学习就结束了，希望能够解决大家的疑惑。理论与实践的搭配能更好的帮助大家学习，快去试试吧！若想继续学习更多相关知识，请继续关注亿速云网站，小编会继续努力为大家带来更多实用的文章！