Bigdata, Hadoop ecosystem, Semantic IoT등의 프로젝트를 진행중에 습득한 내용을 정리하는 곳입니다.
필요한 분을 위해서 공개하고 있습니다.
문의사항은 gooper@gooper.com로 메일을
보내주세요.
* shellscript파일
-----------------
#!/bin/bash
export CLASSPATH=${CLASSPATH}:/opt/cloudera/parcels/CDH-7.1.7-1.cdh7.1.7.p1000.24102687/lib/hive/auxlib/ImpalaJDBC42.jar
export CLASSPATH=${CLASSPATH}:~/gooper/libs/*.jar
kinit -kt /var/lib/keytab/hadoop.keytab hadoop
if [ $? != 0 ]; then
echo "kinit fail"
exit -1
else
echo "kinit sucess"
fi
java ImpalaTest
------------------------
*참고1 : 아래 프로그램을 수행하는데 필요한 jar파일 목록(os폴더 위치 : ~/gooper/libs)
(/opt/cloudera/parcels/CDH/jars에 있으니 복사해서 사용한다, 이곳에은 동일한 jar가 바젼별로 존재하므로 이 경로를 classpath에 통째 포함시켜서 사용하면 실행시 NoSuchMehod같은 오류가 발생한다)
commons-collections-3.2.2.jar hadoop-common-3.1.1.7.1.7.1000-141.jar hive-service-3.1.3000.7.1.7.1000-141.jar log4j-1.2-api-2.17.1.jar
commons-configuration2-2.1.1.jar hive-exec-3.1.3000.7.1.7.1000-141.jar hive-service-rpc-3.1.3000.7.1.7.1000-141.jar log4j-api-2.17.1.jar
commons-lang-2.6.jar hive-jdbc-3.1.3000.7.1.7.1000-141.jar httpclient-4.5.13.jar log4j-core-2.17.1.jar
guava-27.0.1-jre.jar hive-jdbc-3.1.3000.7.1.7.1000-141-standalone.jar jaxb-impl-2.2.3-1.jar slf4j-log4j12-1.7.30.jar
hadoop-auth-3.1.1.7.1.7.1000-141.jar hive-metastore-3.1.3000.7.1.7.1000-141.jar libthrift-0.9.3.jar
commons-lang-2.6.jar hive-jdbc-3.1.3000.7.1.7.1000-141.jar httpclient-4.5.13.jar log4j-core-2.17.1.jar
guava-27.0.1-jre.jar hive-jdbc-3.1.3000.7.1.7.1000-141-standalone.jar jaxb-impl-2.2.3-1.jar slf4j-log4j12-1.7.30.jar
hadoop-auth-3.1.1.7.1.7.1000-141.jar hive-metastore-3.1.3000.7.1.7.1000-141.jar libthrift-0.9.3.jar
* 참고2 : java source(파일명 ImpalaTest.java)
import java.io.IOException;
import java.io.InputStream;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class ImpalaTest {
private static String connectionUrl;
private static String jdbcDriverName;
private static void loadConfiguration() throws IOException {
try {
// connectionUrl = "jdbc:impala://impalajdbcnode.gooper.com1050;AuthMech=3;UID=hadoop;PWD=;UseSasl=0"; // 해당 user로 동작함
connectionUrl = "jdbc:impala://impalajdbc_l4.gooper.com:21051/default;AuthMech=1;KrbRealm=GOOPER.COM;KrbHostFQDN=impala_jdbc_node.gooper.com;KrbServiceName=impala;" ;
//impalaJDBC4.1의 경우
//jdbcDriverName = "com.cloudera.impala.jdbc41.Driver";
//ImpalaJDBC4.2의 경우
jdbcDriverName = "com.cloudera.impala.jdbc.Driver";
} finally {
System.out.println("test");
}
}
public static void main(String[] args) throws IOException, ClassNotFoundException, SQLException {
selectTest();
}
public static void selectTest() throws IOException {
String sqlStatement = "select count(*) as 'abcd' from test.test_table;";
loadConfiguration();
System.out.println("\n=============================================");
System.out.println("Using Connection URL: " + connectionUrl);
Connection con = null;
Statement stmt = null;
try {
Class.forName(jdbcDriverName);
con = DriverManager.getConnection(connectionUrl);
stmt = con.createStatement();
ResultSet rs = stmt.executeQuery(sqlStatement);
System.out.println("\n== Begin Query Results ======================");
System.out.println("rs:"+rs);
// print the results to the console
while (rs.next()) {
// the example query returns one String column
System.out.println(rs.getString(1));
}
System.out.println("== End Query Results =======================\n\n");
} catch (SQLException e) {
e.printStackTrace();
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
stmt.close();
con.close();
} catch (Exception e) {
// swallow
}
}
}
}