Bigdata, Hadoop ecosystem, Semantic IoT등의 프로젝트를 진행중에 습득한 내용을 정리하는 곳입니다.
필요한 분을 위해서 공개하고 있습니다.
문의사항은 gooper@gooper.com로 메일을
보내주세요.
Hive JDBC Connection And JAR file errors
Required jar file
JAR FILE PATH
a. hive-jdbc.jar /usr/lib/hive/lib
b. hive-service /usr/lib/hive/lib
c. libthrift-0.9.2 /usr/lib/hive/lib
d. commons-logging /usr/lib/hive/lib
e. httpclient-4.2.5 /usr/lib/hive/lib
f. httpcore /usr/lib/hive/lib
g. hive-common /usr/lib/hive/lib
h. hive-metastore.jar /usr/lib/hive/lib
i. hadoop-common /usr/lib/hadoop/client-0.20
j. slf4j-api /usr/lib/hadoop/client
k. log4j /usr/lib/hadoop/lib
l. slf4j-log4j /usr/lib/hadoop/lib
Download above JAR file from above listed directory in cloudvm
start beeline
# beeline -u jdbc:hive2://192.168.11.136:10000/default -n admin -d org.apache.hive.jdbc.HiveDriver
# hive --service hiveserver2
Verify hiveserver2 running status
# netstat -nl |grep 10000
If your hive2 is not starting then delete below files from given directory
* Delete : /var/lib/hive/metastore/metastore_db (.lck file) if hiveserver2 not started
Connect to beeline
beeline -u jdbc:hive2://192.168.11.136:10000/default -n admin -d org.apache.hive.jdbc.HiveDriver
or
·
Example
% bin/beeline
Hive version 0.11.0-SNAPSHOT by Apache
beeline> !connect jdbc:hive2://localhost:10000 scott tiger org.apache.hive.jdbc.HiveDriver
Program
package com.kshitish.dbcon;
import java.sql.SQLException;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.Statement;
import java.sql.DriverManager;
/**
* hive2 connection
* start beeline
* command : beeline -u jdbc:hive2://192.168.11.136:10000/default -n admin -d org.apache.hive.jdbc.HiveDriver
* command : hive --service hiveserver2
* command : netstat -nl |grep 10000
* Delete : /var/lib/hive/metastore/metastore_db (.lck file) if hiveserver2 not started
*
* add jar from below directory /usr/lib
*
* -----> hive
* -----> hadoop-mapreduce
* -----> hadoop-0.20-mapreduce
* -----> hadoop
*
* @author k.a.chandra.pradhan
*
*/
public class HiveJdbcClient {
private static String driverName = "org.apache.hive.jdbc.HiveDriver";
public static void main(String[] args) throws SQLException {
try {
Class.forName(driverName);
} catch (ClassNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
System.out.println("Got some exception");
System.exit(1);
}
System.out.println("before connection");
Connection con = DriverManager.getConnection("jdbc:hive2://192.168.11.136:10000/ksh", "", "");
System.out.println("after connection stage");
Statement stmt = con.createStatement();
System.out.println("after connection");
String tableName = "kshitish_storage";
stmt.executeUpdate("CREATE TABLE IF NOT EXISTS " +tableName + "(eid int, name String,salary String, destination String)");
System.out.println("Table have been created");
String sql = "show tables '" + tableName + "'";
System.out.println("Running: " + sql);
Statement stmt1 = con.createStatement();
ResultSet rs = stmt1.executeQuery("describe kshitish_storage");
while (rs.next())
{
System.out.println(rs.getString(1) + ":" + rs.getString(2) );
}
rs.close();
}
}
Possibility of errors during Hive Connection due to JAR file
ERROR WITH JARS
hive-service jar file in /usr/lib/hive/lib
issues
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/hive/service/cli/thrift/TCLIService$Iface
at org.apache.hive.jdbc.HiveDriver.connect(HiveDriver.java:105)
at java.sql.DriverManager.getConnection(Unknown Source)
at java.sql.DriverManager.getConnection(Unknown Source)
at HiveJdbcClientV2.main(HiveJdbcClientV2.java:43)
Caused by: java.lang.ClassNotFoundException: org.apache.hive.service.cli.thrift.TCLIService$Iface
at java.net.URLClassLoader$1.run(Unknown Source)
at java.net.URLClassLoader$1.run(Unknown Source)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(Unknown Source)
at java.lang.ClassLoader.loadClass(Unknown Source)
at sun.misc.Launcher$AppClassLoader.loadClass(Unknown Source)
at java.lang.ClassLoader.loadClass(Unknown Source)
=====================================================================
hive-jdbc.jar in /usr/lib/hive/lib
issues
java.lang.ClassNotFoundException: org.apache.hive.jdbc.HiveDriver
at java.net.URLClassLoader$1.run(Unknown Source)
at java.net.URLClassLoader$1.run(Unknown Source)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(Unknown Source)
at java.lang.ClassLoader.loadClass(Unknown Source)
at sun.misc.Launcher$AppClassLoader.loadClass(Unknown Source)
at java.lang.ClassLoader.loadClass(Unknown Source)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Unknown Source)
at HiveJdbcClientV2.main(HiveJdbcClientV2.java:32)
====================================================================================
libthrift in /usr/libhive/lib
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/thrift/transport/TTransportException
at org.apache.hive.jdbc.HiveDriver.connect(HiveDriver.java:105)
at java.sql.DriverManager.getConnection(Unknown Source)
at java.sql.DriverManager.getConnection(Unknown Source)
at HiveJdbcClientV2.main(HiveJdbcClientV2.java:43)
Caused by: java.lang.ClassNotFoundException: org.apache.thrift.transport.TTransportException
at java.net.URLClassLoader$1.run(Unknown Source)
at java.net.URLClassLoader$1.run(Unknown Source)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(Unknown Source)
at java.lang.ClassLoader.loadClass(Unknown Source)
at sun.misc.Launcher$AppClassLoader.loadClass(Unknown Source)
at java.lang.ClassLoader.loadClass(Unknown Source)
... 4 more
====================================================================================================
commons-logging jar in /usr/libhive/lib
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/commons/logging/LogFactory
at org.apache.hive.jdbc.HiveConnection.<clinit>(HiveConnection.java:94)
at org.apache.hive.jdbc.HiveDriver.connect(HiveDriver.java:105)
at java.sql.DriverManager.getConnection(Unknown Source)
at java.sql.DriverManager.getConnection(Unknown Source)
at HiveJdbcClientV2.main(HiveJdbcClientV2.java:43)
Caused by: java.lang.ClassNotFoundException: org.apache.commons.logging.LogFactory
at java.net.URLClassLoader$1.run(Unknown Source)
at java.net.URLClassLoader$1.run(Unknown Source)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(Unknown Source)
at java.lang.ClassLoader.loadClass(Unknown Source)
at sun.misc.Launcher$AppClassLoader.loadClass(Unknown Source)
at java.lang.ClassLoader.loadClass(Unknown Source)
... 5 more
==============================================================================================
httpclient-4.2.5 in /usr/lib/hive/lib
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/http/client/HttpClient
at org.apache.hive.jdbc.HiveDriver.connect(HiveDriver.java:105)
at java.sql.DriverManager.getConnection(Unknown Source)
at java.sql.DriverManager.getConnection(Unknown Source)
at HiveJdbcClientV2.main(HiveJdbcClientV2.java:43)
Caused by: java.lang.ClassNotFoundException: org.apache.http.client.HttpClient
at java.net.URLClassLoader$1.run(Unknown Source)
at java.net.URLClassLoader$1.run(Unknown Source)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(Unknown Source)
at java.lang.ClassLoader.loadClass(Unknown Source)
at sun.misc.Launcher$AppClassLoader.loadClass(Unknown Source)
at java.lang.ClassLoader.loadClass(Unknown Source)
================================================================================================
httpcore in /usr/lib/hive/lib
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/http/HttpRequestInterceptor
at org.apache.hive.jdbc.HiveDriver.connect(HiveDriver.java:105)
at java.sql.DriverManager.getConnection(Unknown Source)
at java.sql.DriverManager.getConnection(Unknown Source)
at HiveJdbcClientV2.main(HiveJdbcClientV2.java:43)
Caused by: java.lang.ClassNotFoundException: org.apache.http.HttpRequestInterceptor
at java.net.URLClassLoader$1.run(Unknown Source)
at java.net.URLClassLoader$1.run(Unknown Source)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(Unknown Source)
at java.lang.ClassLoader.loadClass(Unknown Source)
at sun.misc.Launcher$AppClassLoader.loadClass(Unknown Source)
at java.lang.ClassLoader.loadClass(Unknown Source)
... 4 more
==================================================================================================
hive-common in /usr/lib/hive/lib
log4j:WARN No appenders could be found for logger (org.apache.hive.jdbc.Utils).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/hadoop/hive/conf/HiveConf
at org.apache.hive.jdbc.HiveConnection.createBinaryTransport(HiveConnection.java:402)
at org.apache.hive.jdbc.HiveConnection.openTransport(HiveConnection.java:193)
at org.apache.hive.jdbc.HiveConnection.<init>(HiveConnection.java:167)
at org.apache.hive.jdbc.HiveDriver.connect(HiveDriver.java:105)
at java.sql.DriverManager.getConnection(Unknown Source)
at java.sql.DriverManager.getConnection(Unknown Source)
at HiveJdbcClientV2.main(HiveJdbcClientV2.java:43)
Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.hive.conf.HiveConf
at java.net.URLClassLoader$1.run(Unknown Source)
at java.net.URLClassLoader$1.run(Unknown Source)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(Unknown Source)
at java.lang.ClassLoader.loadClass(Unknown Source)
at sun.misc.Launcher$AppClassLoader.loadClass(Unknown Source)
at java.lang.ClassLoader.loadClass(Unknown Source)
... 7 more
=======================================================================================================
hive-metastore in /usr/lib/hive/lib
log4j:WARN No appenders could be found for logger (org.apache.hive.jdbc.Utils).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/hadoop/hive/metastore/api/MetaException
at org.apache.hive.jdbc.HiveConnection.createBinaryTransport(HiveConnection.java:402)
at org.apache.hive.jdbc.HiveConnection.openTransport(HiveConnection.java:193)
at org.apache.hive.jdbc.HiveConnection.<init>(HiveConnection.java:167)
at org.apache.hive.jdbc.HiveDriver.connect(HiveDriver.java:105)
at java.sql.DriverManager.getConnection(Unknown Source)
at java.sql.DriverManager.getConnection(Unknown Source)
at HiveJdbcClientV2.main(HiveJdbcClientV2.java:43)
Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.hive.metastore.api.MetaException
at java.net.URLClassLoader$1.run(Unknown Source)
at java.net.URLClassLoader$1.run(Unknown Source)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(Unknown Source)
at java.lang.ClassLoader.loadClass(Unknown Source)
at sun.misc.Launcher$AppClassLoader.loadClass(Unknown Source)
at java.lang.ClassLoader.loadClass(Unknown Source)
... 7 more
====================================================================================================
hadoop-common in /usr/lib/hadoop/client-0.20
log4j:WARN No appenders could be found for logger (org.apache.hive.jdbc.Utils).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/hadoop/conf/Configuration
at org.apache.hive.jdbc.HiveConnection.createBinaryTransport(HiveConnection.java:402)
at org.apache.hive.jdbc.HiveConnection.openTransport(HiveConnection.java:193)
at org.apache.hive.jdbc.HiveConnection.<init>(HiveConnection.java:167)
at org.apache.hive.jdbc.HiveDriver.connect(HiveDriver.java:105)
at java.sql.DriverManager.getConnection(Unknown Source)
at java.sql.DriverManager.getConnection(Unknown Source)
at HiveJdbcClientV2.main(HiveJdbcClientV2.java:43)
Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.conf.Configuration
at java.net.URLClassLoader$1.run(Unknown Source)
at java.net.URLClassLoader$1.run(Unknown Source)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(Unknown Source)
at java.lang.ClassLoader.loadClass(Unknown Source)
at sun.misc.Launcher$AppClassLoader.loadClass(Unknown Source)
at java.lang.ClassLoader.loadClass(Unknown Source)
... 7 more
==============================================================================================
slf4j-api jar in /usr/lib/hadoop/client
log4j:WARN No appenders could be found for logger (org.apache.hive.jdbc.Utils).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
Exception in thread "main" java.lang.NoClassDefFoundError: org/slf4j/LoggerFactory
at org.apache.hive.service.auth.HiveAuthFactory.<clinit>(HiveAuthFactory.java:63)
at org.apache.hive.jdbc.HiveConnection.createBinaryTransport(HiveConnection.java:402)
at org.apache.hive.jdbc.HiveConnection.openTransport(HiveConnection.java:193)
at org.apache.hive.jdbc.HiveConnection.<init>(HiveConnection.java:167)
at org.apache.hive.jdbc.HiveDriver.connect(HiveDriver.java:105)
at java.sql.DriverManager.getConnection(Unknown Source)
at java.sql.DriverManager.getConnection(Unknown Source)
at HiveJdbcClientV2.main(HiveJdbcClientV2.java:43)
Caused by: java.lang.ClassNotFoundException: org.slf4j.LoggerFactory
at java.net.URLClassLoader$1.run(Unknown Source)
at java.net.URLClassLoader$1.run(Unknown Source)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(Unknown Source)
at java.lang.ClassLoader.loadClass(Unknown Source)
at sun.misc.Launcher$AppClassLoader.loadClass(Unknown Source)
at java.lang.ClassLoader.loadClass(Unknown Source)
... 8 more
==================================================================================================
log4j in /usr/lib/hadoop/lib
Dec 15, 2015 8:13:56 PM org.apache.hive.jdbc.Utils parseURL
INFO: Supplied authorities: 192.168.117.136:10000
Dec 15, 2015 8:13:56 PM org.apache.hive.jdbc.Utils parseURL
INFO: Resolved authority: 192.168.117.136:10000
Failed to instantiate SLF4J LoggerFactory
Reported exception:
java.lang.NoClassDefFoundError: org/apache/log4j/Level
at org.slf4j.LoggerFactory.bind(LoggerFactory.java:128)
at org.slf4j.LoggerFactory.performInitialization(LoggerFactory.java:107)
at org.slf4j.LoggerFactory.getILoggerFactory(LoggerFactory.java:295)
at org.slf4j.LoggerFactory.getLogger(LoggerFactory.java:269)
at org.slf4j.LoggerFactory.getLogger(LoggerFactory.java:281)
at org.apache.hive.service.auth.HiveAuthFactory.<clinit>(HiveAuthFactory.java:63)
at org.apache.hive.jdbc.HiveConnection.createBinaryTransport(HiveConnection.java:402)
at org.apache.hive.jdbc.HiveConnection.openTransport(HiveConnection.java:193)
at org.apache.hive.jdbc.HiveConnection.<init>(HiveConnection.java:167)
at org.apache.hive.jdbc.HiveDriver.connect(HiveDriver.java:105)
at java.sql.DriverManager.getConnection(Unknown Source)
at java.sql.DriverManager.getConnection(Unknown Source)
at HiveJdbcClientV2.main(HiveJdbcClientV2.java:43)
Caused by: java.lang.ClassNotFoundException: org.apache.log4j.Level
at java.net.URLClassLoader$1.run(Unknown Source)
at java.net.URLClassLoader$1.run(Unknown Source)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(Unknown Source)
at java.lang.ClassLoader.loadClass(Unknown Source)
at sun.misc.Launcher$AppClassLoader.loadClass(Unknown Source)
at java.lang.ClassLoader.loadClass(Unknown Source)
... 13 more
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/log4j/Level
at org.slf4j.LoggerFactory.bind(LoggerFactory.java:128)
at org.slf4j.LoggerFactory.performInitialization(LoggerFactory.java:107)
at org.slf4j.LoggerFactory.getILoggerFactory(LoggerFactory.java:295)
at org.slf4j.LoggerFactory.getLogger(LoggerFactory.java:269)
at org.slf4j.LoggerFactory.getLogger(LoggerFactory.java:281)
at org.apache.hive.service.auth.HiveAuthFactory.<clinit>(HiveAuthFactory.java:63)
at org.apache.hive.jdbc.HiveConnection.createBinaryTransport(HiveConnection.java:402)
at org.apache.hive.jdbc.HiveConnection.openTransport(HiveConnection.java:193)
at org.apache.hive.jdbc.HiveConnection.<init>(HiveConnection.java:167)
at org.apache.hive.jdbc.HiveDriver.connect(HiveDriver.java:105)
at java.sql.DriverManager.getConnection(Unknown Source)
at java.sql.DriverManager.getConnection(Unknown Source)
at HiveJdbcClientV2.main(HiveJdbcClientV2.java:43)
Caused by: java.lang.ClassNotFoundException: org.apache.log4j.Level
at java.net.URLClassLoader$1.run(Unknown Source)
at java.net.URLClassLoader$1.run(Unknown Source)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(Unknown Source)
at java.lang.ClassLoader.loadClass(Unknown Source)
at sun.misc.Launcher$AppClassLoader.loadClass(Unknown Source)
at java.lang.ClassLoader.loadClass(Unknown Source)
... 13 more
====================================================================================================================
slf4j-log4j in /usr/lib/hadoop/lib
only warning u will get but it will execute
SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
SLF4J: Defaulting to no-operation (NOP) logger implementation
SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.