[Scala] Scala操作Hive

 

环境–spark-1.5.1  scala-2.10.4

 

package com.huawei.bigdata.spark.examples

import org.apache.hadoop.conf.Configuration
import org.apache.spark.{SparkContext, SparkConf}
import org.apache.spark.rdd.RDD
import org.apache.spark.SparkContext._

import com.huawei.hadoop.security.LoginUtil

object scalaHive {
  def main (args: Array[String]) {
    
    if (args.length < 1) {
      System.err.println("Usage: CollectFemaleInfo <file>")
      System.exit(1)
    }
    /**STEP1: Variables Initialized**/
    val userPrincipal = "ebda"
    val userKeytabPath = "/home/ap/dip/appjob/shelljob/dzyh/jobScala/user.keytab"
    val krb5ConfPath = "/home/ap/dip/appjob/shelljob/dzyh/jobScala/krb5.conf"
    val hadoopConf: Configuration  = new Configuration()
    LoginUtil.login(userPrincipal, userKeytabPath, krb5ConfPath, hadoopConf);
    
    // Configure the Spark application name.
    val conf = new SparkConf().setAppName(args(0))
    val sc=new SparkContext(conf)

    //create hivecontext
    val sqlContext = new org.apache.spark.sql.hive.HiveContext(sc)

    /**STEP3: Data Analysis**/

    sqlContext.sql("CREATE TABLE IF NOT EXISTS tbl_scala_hbase (key INT, value STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' ")  
    sqlContext.sql("insert into tbl_scala_hbase values(1,'Zacks')");
    sqlContext.sql(" SELECT * FROM tbl_scala_hbase").collect().foreach(println)
    sc.stop()

  }
}
分类上一篇:无,已是最新文章    分类下一篇:

Leave a Reply