2017. 8. 1. 18:12ㆍ서버 프로그래밍
<Hive 설치>
$ sudo yum install mysql mysql-server
$ sudo chkconfig mysqld on
$ sudo service mysqld start
$ sudo mysql_secure_installation
$ vi /etc/my.cnf
[mysqld]
bind-address=127.0.0.1
$ wget HYPERLINK "http://apache.mirror.cdnetworks.com/hive/stable-2/apache-hive-2.1.1-bin.tar.gz"http://apache.mirror.cdnetworks.com/hive/stable-2/apache-hive-2.1.1-bin.tar.gz
$ tar xzvf apache-hive-2.1.1-bin.tar.gz
$ ln -s apache-hive-2.1.1-bin hive
$ chmod -R 775 ~/apache-hive-2.1.1-bin
$ vi .bashrc
############### HIVE
export HIVE_HOME=/home/eduuser/apache-hive-2.1.1-bin
export PATH=$PATH:$HIVE_HOME/bin
export CLASSPATH=${JAVA_HOME}/lib:${JREHOME}/lib:/home/eduuser/apache-hive-2.1.1-bin/lib
############### HIVE
$ source .bashrc
$ cd hive
$ cd conf
$ cp hive-env.sh.template hive-env.sh
$ cp hive-default.xml.template hive-site.xml
$ vi hive-env.sh
HADOOP_HOME=/home/eduuser/hadoop-2.7.1
$ vi hive-site.xml
<property>
<name>hive.cli.print.current.db</name>
<value>true</value>
</property>
<property>
<name>javax.jdo.option.ConnectionURL</name>
<value>jdbc:mysql://127.0.0.1:3306/hive?createDatabaseIfNotExist=true</value>
<description>
JDBC connect string for a JDBC metastore.
To use SSL to encrypt/authenticate the connection, provide database-specific SSL flag in the connection URL.
For example, jdbc:postgresql://myhost/db?ssl=true for postgres database.
</description>
</property>
<property>
<name>javax.jdo.option.ConnectionDriverName</name>
<value>com.mysql.jdbc.Driver</value>
<description>Driver class name for a JDBC metastore</description>
</property>
<property>
<name>javax.jdo.option.ConnectionUserName</name>
<value>root</value>
<description>Username to use against metastore database</description>
</property>
<property>
<name>javax.jdo.option.ConnectionPassword</name>
<value>hello.edu</value>
<description>password to use against metastore database</description>
</property>
<property>
<name>hive.exec.local.scratchdir</name>
<value>/home/eduuser/iotmp</value>
<description>Local scratch space for Hive jobs</description>
</property>
<property>
<name>hive.downloaded.resources.dir</name>
<value>/home/eduuser/iotmp</value>
<description>Temporary local directory for added resources in the remote file system.</description>
</property>
$ cd ~
$ mkdir iotmp
$ chmod -R 775 iotmp
$ wget HYPERLINK "http://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.38.tar.gz"http://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.38.tar.gz
$ tar xzvf mysql-connector-java-5.1.38.tar.gz
$ cp mysql-connector-java-5.1.38-bin.jar /home/eduuser/apache-hive-2.1.1-bin/lib
$ hdfs dfs -mkdir /tmp
$ hdfs dfs -chmod g+w /tmp
$ hdfs dfs -mkdir /user/eduuser/warehouse
$ hdfs dfs -chmod g+w /user/eduuser/warehouse
$ schematool -initSchema -dbType mysql
which: no hbase in (/usr/java/jdk1.8.0_65/bin:/usr/lib64/qt-3.3/bin:/bin:/usr/bin:/usr/X11R6/bin:/usr/local/bin:/usr/java/jdk1.8.0_65:/home/eduuser/apache-hive-2.1.1-bin/bin:/home/eduuser/hadoop-2.7.1/sbin:/home/eduuser/hadoop-2.7.1/bin)
SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/home/eduuser/apache-hive-2.1.1-bin/lib/log4j-slf4j-impl-2.4.1.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/home/eduuser/hadoop-2.7.1/share/hadoop/common/lib/slf4j-log4j12-1.7.10.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.apache.logging.slf4j.Log4jLoggerFactory]
Metastore connection URL: jdbc:mysql://127.0.0.1:3306/hive?createDatabaseIfNotExist=true
Metastore Connection Driver : com.mysql.jdbc.Driver
Metastore connection User: root
Starting metastore schema initialization to 2.1.0
Initialization script hive-schema-2.1.0.mysql.sql
Initialization script completed
schemaTool completed
$ hive
which: no hbase in (/usr/java/jdk1.8.0_65/bin:/usr/lib64/qt-3.3/bin:/bin:/usr/bin:/usr/X11R6/bin:/usr/local/bin:/usr/java/jdk1.8.0_65:/home/eduuser/apache-hive-2.1.1-bin/bin:/home/eduuser/hadoop-2.7.1/sbin:/home/eduuser/hadoop-2.7.1/bin)
SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/home/eduuser/apache-hive-2.1.1-bin/lib/log4j-slf4j-impl-2.4.1.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/home/eduuser/hadoop-2.7.1/share/hadoop/common/lib/slf4j-log4j12-1.7.10.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.apache.logging.slf4j.Log4jLoggerFactory]
Logging initialized using configuration in jar:file:/home/eduuser/apache-hive-2.1.1-bin/lib/hive-common-2.1.1.jar!/hive-log4j2.properties Async: true
Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases.
hive>
hive> CREATE DATABASE word_database;
hive> USE word_database;
hive> CREATE TABLE wtable (line STRING);
hive> LOAD DATA LOCAL INPATH 'input.txt' OVERWRITE INTO TABLE wtable;
Loading data to table word_database.wtable
OK
Time taken: 0.956 seconds
hive> CREATE TABLE word_count AS SELECT word,count(1) AS count FROM (SELECT explode(split(line,' ')) AS word FROM wtable) w GROUP BY word ORDER BY word;
..................................
MapReduce Jobs Launched:
Stage-Stage-1: Map: 1 Reduce: 1 Cumulative CPU: 5.23 sec HDFS Read: 7992 HDFS Write: 186 SUCCESS
Stage-Stage-2: Map: 1 Reduce: 1 Cumulative CPU: 3.62 sec HDFS Read: 5124 HDFS Write: 106 SUCCESS
Total MapReduce CPU Time Spent: 8 seconds 850 msec
OK
Time taken: 53.807 seconds
hive> select * from word_count;
OK
a 2
book 2
read 1
write 1
Time taken: 0.175 seconds, Fetched: 4 row(s)
<Hbase 설치>
$ hdfs dfs -df -h
17/07/31 15:16:09 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
Filesystem Size Used Available Use%
hdfs://localhost:9000 76.4 G 11.3 G 34.8 G 15%
$ ln -s hbase-1.2.6 hbase
$ vi .bashrc
export HBASE_HOME=/home/eduuser/hbase-1.2.6
export PATH=$PATH:$HBASE_HOME/bin
export HBASE_PID_DIR=/home/eduuser/hbase-1.2.6/pid
$ sudo vi /etc/bashrc
JAVA_HOME=/usr/java/jdk1.8.0_65
HBASE_PID_DIR=/home/eduuser/hbase-1.2.6/pid
$ source .bashrc
$ vi hbase-site.xml
<property>
<name>hbase.rootdir</name>
<value>hdfs://127.0.0.1:9000/hbase</value>
</property>
<property>
<name>hbase.cluster.distributed</name>
<value>false</value>
</property>
<property>
<name>hbase.zookeeper.quorum</name>
<value>localhost</value>
</property>
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
<property>
<name>hbase.zookeeper.property.clientPort</name>
<value>2181</value>
</property>
$ vi hbase-env.sh
export HBASE_MANAGES_ZK=false
export HBASE_PID_DIR=/home/eduuser
$ hdfs dfsadmin -safemode leave
17/07/31 15:21:44 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
Safe mode is OFF
$ start-hbase.sh
starting master, logging to /home/eduuser/hbase-1.2.6/logs/hbase-eduuser-master-localhost.localdomain.out
Java HotSpot(TM) 64-Bit Server VM warning: ignoring option PermSize=128m; support was removed in 8.0
Java HotSpot(TM) 64-Bit Server VM warning: ignoring option MaxPermSize=128m; support was removed in 8.0
$ jps
785 NameNode
3169 NodeManager
1874 DataNode
803 org.eclipse.equinox.launcher_1.3.0.v20140415-2008.jar
2468 SecondaryNameNode
18537 HMaster
3018 ResourceManager
18779 Jps
$ hbase shell
2017-07-31 15:28:18,705 WARN [main] util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/home/eduuser/hbase-1.2.6/lib/slf4j-log4j12-1.7.5.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/home/eduuser/hadoop-2.7.1/share/hadoop/common/lib/slf4j-log4j12-1.7.10.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]
HBase Shell; enter 'help<RETURN>' for list of supported commands.
Type "exit<RETURN>" to leave the HBase Shell
Version 1.2.6, rUnknown, Mon May 29 02:25:32 CDT 2017
hbase(main):001:0> list
TABLE
0 row(s) in 0.4860 seconds
=> []
hbase(main):002:0> create 'emps', 'personal data', 'professional data'
0 row(s) in 1.3240 seconds
=> Hbase::Table - emps
hbase(main):003:0> list
TABLE
emps
1 row(s) in 0.0110 seconds
=> ["emps"]
hbase(main):004:0> put 'emps', '1', 'personal data:name', 'hong'
0 row(s) in 0.3030 seconds
hbase(main):005:0> put 'emps', '1', 'personal data:city', 'pusan'
0 row(s) in 0.0240 seconds
hbase(main):006:0> put 'emps', '1', 'professional data:designation','manager'
0 row(s) in 0.0300 seconds
hbase(main):007:0> put 'emps', '1', 'professional data:salary', '50000'
0 row(s) in 0.0170 seconds
hbase(main):008:0> scan 'emps'
ROW COLUMN+CELL
1 column=personal data:city, timestamp=1501482658232, value=pus
an
1 column=personal data:name, timestamp=1501482642936, value=hon
g
1 column=professional data:designation, timestamp=1501482679590
, value=manager
1 column=professional data:salary, timestamp=1501482699836, val
ue=50000
1 row(s) in 0.0670 seconds