当前位置: 动力学知识库 > 问答 > 编程问答 >

scala - java.lang.IllegalArgumentException: Wrong FS: file:/tmp/spark-d48e6576/scratch_hive_2016-12-28_10-47-18_519_1, expected: hdfs://nameservice1

问题描述:

I am trying to parse JSON log file with spark and append to external hive table.

def main(args: Array[String]) {

val local: String = args(0)

val sparkConf: SparkConf = new SparkConf().setAppName("Proxy")

val ctx = new SparkContext(sparkConf)

val sqlContext = new HiveContext(ctx)

import sqlContext.implicits._

val df = sqlContext.read.schema(schema).json("group-8_instance-48_2016-10- 19-16-54.log")

df.registerTempTable("proxy_par_tmp")

val results_transaction =sqlContext.sql("SELECT type,time,path,protocol,protocolSrc, duration, status, serviceContexts,customMsgAtts,correlationId, legs FROM proxy_par_tmp where type='transaction'")

val two = saveFile(results_transaction).toDF()

two.write.mode(SaveMode.Append).saveAsTable("lzwpp_ushare.proxy_par")

ctx.stop

}

def saveFile(file:DataFrame): RDD[String]={

val five =file.map { t =>( t(0).toString + "~" +t(1).toString + "~" +t(2).toString + "~" +t(3).toString + "~" +t(4).toString

+ "~" +t(5).toString + "~" +t(6).toString + "~" +t(7).toString + "~" +t(8).toString + "~" +t(9).toString

+ "~"+ t(10).toString) }

five

}

Also tried with

 two.saveAsTable("lzwpp_ushare.proxy_par",SaveMode.Append)

The problem is when I try the same thing with Hive managed table it works fine but not for External table.

Spark-shell as well as spark-submit throws the same error.

It writes to HDFS

Thanks.

分享给朋友:
您可能感兴趣的文章:
随机阅读: