[ https://issues.apache.org/jira/browse/HIVE-25647?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
St Li updated HIVE-25647: ------------------------- Description: master,slave1,slave2master,slave1,slave2//opt represent wechat hadoop bigdata dev//2019 :bigdata competitionhadoop 50070hbase 16010storm 8080 #hostnamehostnamectl set-hostname master && bash hostname master && bash hostname slave1/slave2 && bash vim /etc/hostname master/slave1/slave2vim /etc/hosts ip master ip slave1 ipslave2 #yumcd /etc/yum.repos.d && rm -rf *wget http://172.16.47.240/bigdata/repofile/bigdata.repoyum clean all #firewallsystemctl stop firewalldsystemctl status firewalld #timezonetzselect 5-9-1-1echo "TZ='Asia/Shanghai'; export TZ" >> /etc/profile && source /etc/profile #ntpyum install -y ntpvim /etc/ntp.conf//#server 0~3.centos.pool.ntp.org iburstserver 127.127.1.0fudge 127.127.1.0 stratum 10/bin/systemctl restart ntpd.servicentpdate master (slave1,slave2) #crontabservice crond status/sbin/service crond startcrontab -e*/30 8-17 * * * /usr/sbin/ntpdate mastercrontab –l #ssh passwordssh-keygen -t dsa -P '' -f ~/.ssh/id_dsacat /root/.ssh/id_dsa.pub >> /root/.ssh/authorized_keysscp ~/.ssh/authorized_keys root@slave1:~/.ssh/scp ~/.ssh/authorized_keys root@slave2:~/.ssh/ ssh-copy-id masterssh-copy-id slave1ssh-copy-id slave2 #install jdkmkdir -p /usr/javatar -zxvf jdk-8u171-linux-x64.tar.gz -C /usr/java/ vim /etc/profileexport JAVA_HOME=/usr/java/jdk1.8.0_171export CLASSPATH=$JAVA_HOME/lib/export PATH=$PATH:$JAVA_HOME/bin source /etc/profile && java -version scp -r /usr/java root@slave1:/usr/scp -r /usr/java root@slave2:/usr/ #install hadoopmkdir -p /usr/hadoop && cd /usr/hadooptar -zxvf /usr/hadoop/hadoop-2.7.3.tar.gz -C /usr/hadoop/rm -rf /usr/hadoop/hadoop-2.7.3.tar.gzvim /etc/profileexport HADOOP_HOME=/usr/hadoop/hadoop-2.7.3export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbinhadoop //test hadoop-env.sh/mapred-env.sh/yarn-en.shexport JAVA_HOME=/usr/java/jdk1.8.0_171 ##vim core-site.xmlfs.default.name \{hdfs://master:9000}hadoop.tmp.dir \{/usr/hadoop/hadoop-2.7.3/hdfs/tmp}io.file.buffer.size \{131072}fs.checkpoint.period \{60}fs.checkpoint.size \{67108864} ##hdfs-site.xmldfs.replication \{2}dfs.namenode.name.dir \{file:/usr/hadoop/hadoop-2.7.3/hdfs/name}dfs.datanode.data.dir \{file:/usr/hadoop/hadoop-2.7.3/hdfs/data} ##vim yarn-env.shyarn.resourcemanager.address \{master:18040}yarn.resourcemanager.scheduler.address \{master:18030}yarn.resourcemanager.webapp.address \{master:18088}yarn.resourcemanager.resource-tracker.address \{18025}yarn.resourcemanager.admin.address \{master:18141}yarn.nodemanager.aux-services \{mapreduce_shuffle}yarn.nodemanager.auxservices.mapreduce.shuffle.class \{org.apache.hadoop.mapred.ShuffleHandler} #vim mapred-site.xmlmapreduce.framework.name \{yarn} #slaves fileecho master > master && echo slave1 > slaves && echo slave2 >> slaves #hadoop formathadoop namenode -format (master) //has been successfully#start hadoopstart-all.shmaster :NameNode,SecondaryNameNode,ResourceManagerslave1~2:DataNode,NodeManager start-dfs.shstart-yarn.shhadoop-daemon.sh start namenodehadoop-daemon.sh start datanodehadoop-daemon.sh start secondarynamenodehadoop-daemon.sh start resourcemanagerhadoop-daemon.sh start nodemanager test hdfs& mapreducehadoop fs -mkdir /inputhadoop fs -put $HADOOP_HOME/README.txt /input http://master:50070hadoop jar $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.8.4.jar #install hiveyum -y install mysql-community-server slave2:mysqlserver slave1:hiveserver master:hiveclientsystemctl daemon-reloadsystemctl start mysqldcat /var/log/mysqld.log grep "temporary password"mysql -uroot -pset global validate_password_policy=0;set global validate_password_length=4;alter user 'root'@'localhost' identified by '123456';mysql -uroot -p123456create user 'root'@'%' identified by '123456';grant all privileges on *.* to 'root'@'%' with grant option;flush privileges; mkdir -p /usr/hive tar -zxvf /usr/hive/apache-hive-2.1.1-bin.tar.gz -C /usr/hive/ vim /etc/profile //for hiveexport HIVE_HOME=/usr/hive/apache-hive-2.1.1-binexport PATH=$PATH:$HIVE_HOME/binsource /etc/profile vim hive-env.shcd $HIVE_HOME/conf && vim hive-env.shexport HADOOP_HOME=/usr/hadoop/hadoop-2.7.3export HIVE_CONF_DIR=/usr/hive/apache-hive-2.1.1-bin/confexport HIVE_AUX_JARS_PATH=/usr/hive/apache-hive-2.1.1-bin/lib cp $HIVE_HOME/lib/jline-2.12.jar $HADOOP_HOME/share/hadoop/yarn/lib/ ##slave1 hive-servercd $HIVE_HOME/lib && wget or cp mysql-connector-java-5.1.47-bin.jar hive-site.xml (hive-server)hive.metastore.warehouse.dir \{/user/hive_remote/warehouse}javax.jdo.option.ConnectionDriverName \{com.mysql.jdbc.Driver}javax.jdo.option.ConnectionURL \{jdbc:mysql://slave2:3306/hive?createDatabaseIfNotExist=true&useSSL=false}javax.jdo.option.ConnectionUserName \{root}javax.jdo.option.ConnectionPassword \{123456} hive-site.xml (hive client)hive.metastore.warehouse.dir \{/user/hive_remote/warehouse}hive.metastore.local \{false}hive.metastore.uris \{thrift://slave1:9083} //opt hadoop fs -mkdir /tmp hadoop fs -mkdir -p /user/hive/warehousehadoop fs -chmod g+w /tmp& /user/hive/warehouse//opt modify log locationvim hive-log4j2.propertiesproperty.hive.log.dir=/home/hadoop //opt hive-site.xmlhive.cli.print.current.db \{true}hive.cli.print.header \{true} beeline connect shoud config core-site.xmlhadoop.proxyuser.hadoop.hosts \{*}..groups \{*}bin/beeline->!connect jdbc:hive2://centos01:10000or bin/beeline -u jdbc:hive2://centos01:10000 -n hadoop desc table/desc database/desc database extended testdb; hive-serverschematool -dbType mysql -initSchemahive --service metastore hive-clienthive \ncreate database hongyaa; #install zookeepermkdir -p /usr/zookeeper && cd /usr/zookeepertar -zxvf /usr/zookeeper/zookeeper-3.4.10.tar.gz -C /usr/zookeeper/vi /etc/profile //for zookeeperexport ZOOKEEPER_HOME=/usr/zookeeper/zookeeper-3.4.10PATH=$PATH:$ZOOKEEPER_HOME/binsource /etc/profile cd /usr/zookeeper/zookeeper-3.4.10/conf/ && mv zoo_sample.cfg zoo.cfg && vimzoo.cfgtickTime=2000initLimit=10syncLimit=5dataDir=/usr/zookeeper/zookeeper-3.4.10/zkdataclientPort=2181dataLogDir=/usr/zookeeper/zookeeper-3.4.10/zkdatalogserver.1=master:2888:3888server.2=slave1:2888:3888server.3=slave2:2888:3888 cd /usr/zookeeper/zookeeper-3.4.10 && mkdir zkdata zkdatalogcd /usr/zookeeper/zookeeper-3.4.10/zkdata && echo "1" >> myid zkServer.sh start(master,slave1,slave2)netstat -antp|grep 2181 #install hbase//opttar -zxf hbase-1.2.6.1-bin.tar.gz -C /opt/modules/export JAVA_HOME=/usr/java/jdk1.8.0_171export HBASE_MANAGES_ZK=true vim hbase-site.xmlhbase.rootdir \{hdfs://centos01:9000}hbase.cluster.distributed \{true}hbase.zookeeper.quorum \{centos01,centos02,centos03}hbase.zookeeper.property.dataDir \{/opt/modules/hbase-1.2.6/zkData} #install kafkatar -zxvf kafka_2.11-2.0.0.tgz -C /opt/modulesvim config/server.propertiesbroker.id=1num.partitions=2default.replication.factor=1lisheners=PLAINTEXT://centos01:9092log.dirs=/opt/modules/kafka_2.11-2.0.0/kafka-logszookeeper.connect=centos01:2181,centos02:2181,centos03:2181scp to centos02,centos03 and modify id bin/kafka-server-start.sh -daemon config/server.properties/kafka-server-start.sh -daemon config/server.properties create topicbin/kafka-topics.sh \--create \--zookeeper centos01:2181,centos02:2181,centos03:2181 \--replication-factor 2 \--partitions 2 \--topic topictest bin/kafka-topics.sh \--list \--zookeeper centos01:2181 bin/kafka-topics.sh \--describe \--zookeeper centos01:2181//生产者bin/kafka-console-producer.sh \--broker-list centos01:9092,centos02:9092,centos03:9092 \--topic topictest//消费者bin/kafka-console-consumer.sh \--bootstrap-server centos01:9092,centos02:9092,centos03:9092 \--topic topictest java -Xms256m -Xmx512m -classpath .:./lib/* com.cmcc.hsd.loader.ScannerV2p9Loader -t 6 -d 202008250000 #install stormtar zxvf apache-storm-1.1.0.tar.gz -C /opt/modulesmv apache-storm-1.1.0/ storm-1.1.0vim /etc/profileexport STORM_HOME=/opt/modules/storm-1.1.0export PATH=$PATH:$STORM_HOME/bin storm-env.shexport JAVA_HOME=/opt/softwares/jdk1.8.0_201export STORM_CONF_DIR="/opt/modules/storm-1.1.0/conf"vim storm.yaml storm.zookeeper.servers: - "centos01" - "centos02" - "centos03" supervisor.slots.ports: - 6700 - 6701 - 6702 - 6703 storm.zookeeper.port: 2181 storm.local.dir: "/opt/modules/storm-1.1.0/data" scp -r to centos01,centos02,centos03 storm nimbus > /dev/null 2>&1 & (master)storm ui > /dev/null 2>&1 & storm supervisor > /dev/null 2>&1 & (slave) jpsnimbus/Supervisor #install sparktar zxvf spark-2.4.0-bin-hadoop2.7.tgz -C /opt/modulesvim conf/slavescentos02centos03vim spark-env.shexport JAVA_HOME=/opt/softwares/jdk1.8.0_201export SPARK_MASTER_IP=centos01export SPARK_MASTER_PORT=7077 export HADOOP_HOME=/opt/modules/hadoop-2.8.4export HADOOP_CONF_DIR=$HADOOP_HOME/etc/haoop scp -r to other node spark ha delete SPARK_MASTER_IP add export SPARK_DAEMON_JAVA_OPT="-Dspark.deploy.recoveryMode=ZOOKEEPER-Dspark.deploy.zookeeper.url=centos01:2181,centos02:2181,centos03:2181-Dspark.deploy.zookeepr.dir=/spark" bin/spark-submit \--class org.apache.spark.examples.SparkPi \--master yarn \--deploy-mode cluster \/opt/modules/spark-2.4.0-bin-hadoop2.7/examples/jars/spark-examples_2.11-2.4.0.jar bin/spark-shell \--master spark://centos02:7077 bin/spark-submit \--class org.apache.spark.examples.SparkPi \--master yarn \--deploy-mode cluster \--drive-memory 512m \--executor-memory 1g \--executor-cores 2 \/opt/modules/spark-2.4.0-bin-hadoop2.7/examples/jars/spark-examples_2.11-2.4.0.jar yarn-site.xml ofen kill by yarnpmem-check-enablevmem-check-enable #analysis data *****create database hive;use hive;create table bike (duration int,startdate timestamp,enddate timestamp,type string) row format delimitedfields terminated by ','; LOAD DATA LOCAL INPATH './load.csv' INTO TABLE loan;insert overwrite local directory '/home/hadoop/college001/'row format delimited fields terminated by 't'select count(*) from bike;##about hiveDateType ,UDAF UDTFselect from_unixtime cast(concat(startdate,':00') as timestamp),'yyyyMMdd000000') from bike limit 10;select date_format('2019-10-07', 'yyyyMMdd000000')select from_unixtime(unix_timestamp('2019-10-07', 'yyyy-MM-dd'), 'yyyyMMdd000000')select unix_timestamp('2019-10-07 13:24:20','yyyy-MM-dd HH:mm:ss') JSON.stringify(option) iconv -f gbk -t utf8 a.txt –o b.txt mapreduce output -rw-r--r-- 2 hadoop supergroup 0 2021-10-26 07:06 /output2/_SUCCESS-rw-r--r-- 2 hadoop supergroup 1306 2021-10-26 07:06 /output2/part-r-00000 #add node slave3yum install -y ntp && ntpdate mastervim /etc/hostsslave3scp ~/.ssh/authorized_keys root@slave3:~/.ssh/yum install -y ntp && ntpdate mastervim /etc/profile && source /etc/profileexport JAVA...export HADOOP...vim $HADOOP_HOME/etc/hadoop/slaves add slave3 $HADOOP_HOME/sbin/hadoop-daemon.sh start datanode$HADOOP_HOME/sbin/yarn-daemon.sh start nodemanagerhdfs dfsadmin -refreshNodessbin/start-balancer.shhdfs dfsadmin -report echo "slave3" > excludes hdfs-site.xml (master) dfs.hosts.exclude \{/usr/hadoop/hadoop-2.7.3/etc/hadoop/excludes} hdfs dfsadmin -refreshNodeshdfs dfsadmin -reporthadoop-daemon.sh stop datanodeyarn-daemon.sh stop nodemanagerhdfs dfsadmin -reportsbin/start-balancer.sh was: do not care this just test close firewall:systemctl stop firewalld check :systemctl status firewalld chose:tzselect echo "TZ='Asia/Shanghai'; export TZ" >> /etc/profile && source /etc/profile yum install -y ntp vim /etc/ntp.conf #server0-3 add : fudge 127.127.1.0 stratum 10 /bin/systemctl restart ntpd.service ntpdate master //slaveshang service crond status /sbin/service crond start > hadoop memo > ----------- > > Key: HIVE-25647 > URL: https://issues.apache.org/jira/browse/HIVE-25647 > Project: Hive > Issue Type: Wish > Components: Configuration > Affects Versions: 3.1.2 > Environment: hadoop 2.7.3 > Reporter: St Li > Assignee: St Li > Priority: Major > Fix For: All Versions > > Attachments: worldip.csv > > > master,slave1,slave2master,slave1,slave2//opt represent wechat hadoop bigdata > dev//2019 :bigdata competitionhadoop 50070hbase 16010storm 8080 > #hostnamehostnamectl set-hostname master && bash hostname master && bash > hostname slave1/slave2 && bash vim /etc/hostname master/slave1/slave2vim > /etc/hosts ip master ip slave1 ipslave2 > #yumcd /etc/yum.repos.d && rm -rf *wget > http://172.16.47.240/bigdata/repofile/bigdata.repoyum clean all > #firewallsystemctl stop firewalldsystemctl status firewalld > #timezonetzselect 5-9-1-1echo "TZ='Asia/Shanghai'; export TZ" >> > /etc/profile && source /etc/profile > #ntpyum install -y ntpvim /etc/ntp.conf//#server 0~3.centos.pool.ntp.org > iburstserver 127.127.1.0fudge 127.127.1.0 stratum 10/bin/systemctl restart > ntpd.servicentpdate master (slave1,slave2) > #crontabservice crond status/sbin/service crond startcrontab -e*/30 8-17 * * > * /usr/sbin/ntpdate mastercrontab –l > #ssh passwordssh-keygen -t dsa -P '' -f ~/.ssh/id_dsacat > /root/.ssh/id_dsa.pub >> /root/.ssh/authorized_keysscp ~/.ssh/authorized_keys > root@slave1:~/.ssh/scp ~/.ssh/authorized_keys root@slave2:~/.ssh/ > ssh-copy-id masterssh-copy-id slave1ssh-copy-id slave2 > #install jdkmkdir -p /usr/javatar -zxvf jdk-8u171-linux-x64.tar.gz -C > /usr/java/ > vim /etc/profileexport JAVA_HOME=/usr/java/jdk1.8.0_171export > CLASSPATH=$JAVA_HOME/lib/export PATH=$PATH:$JAVA_HOME/bin > source /etc/profile && java -version > scp -r /usr/java root@slave1:/usr/scp -r /usr/java root@slave2:/usr/ > #install hadoopmkdir -p /usr/hadoop && cd /usr/hadooptar -zxvf > /usr/hadoop/hadoop-2.7.3.tar.gz -C /usr/hadoop/rm -rf > /usr/hadoop/hadoop-2.7.3.tar.gzvim /etc/profileexport > HADOOP_HOME=/usr/hadoop/hadoop-2.7.3export > PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbinhadoop //test > hadoop-env.sh/mapred-env.sh/yarn-en.shexport JAVA_HOME=/usr/java/jdk1.8.0_171 > ##vim core-site.xmlfs.default.name \{hdfs://master:9000}hadoop.tmp.dir > \{/usr/hadoop/hadoop-2.7.3/hdfs/tmp}io.file.buffer.size > \{131072}fs.checkpoint.period \{60}fs.checkpoint.size \{67108864} > ##hdfs-site.xmldfs.replication \{2}dfs.namenode.name.dir > \{file:/usr/hadoop/hadoop-2.7.3/hdfs/name}dfs.datanode.data.dir > \{file:/usr/hadoop/hadoop-2.7.3/hdfs/data} > ##vim yarn-env.shyarn.resourcemanager.address > \{master:18040}yarn.resourcemanager.scheduler.address > \{master:18030}yarn.resourcemanager.webapp.address > \{master:18088}yarn.resourcemanager.resource-tracker.address > \{18025}yarn.resourcemanager.admin.address > \{master:18141}yarn.nodemanager.aux-services > \{mapreduce_shuffle}yarn.nodemanager.auxservices.mapreduce.shuffle.class > \{org.apache.hadoop.mapred.ShuffleHandler} > #vim mapred-site.xmlmapreduce.framework.name \{yarn} > #slaves fileecho master > master && echo slave1 > slaves && echo slave2 >> > slaves > #hadoop formathadoop namenode -format (master) //has been > successfully#start hadoopstart-all.shmaster > :NameNode,SecondaryNameNode,ResourceManagerslave1~2:DataNode,NodeManager > start-dfs.shstart-yarn.shhadoop-daemon.sh start namenodehadoop-daemon.sh > start datanodehadoop-daemon.sh start secondarynamenodehadoop-daemon.sh start > resourcemanagerhadoop-daemon.sh start nodemanager > test hdfs& mapreducehadoop fs -mkdir /inputhadoop fs -put > $HADOOP_HOME/README.txt /input > http://master:50070hadoop jar > $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.8.4.jar > #install hiveyum -y install mysql-community-server slave2:mysqlserver > slave1:hiveserver master:hiveclientsystemctl daemon-reloadsystemctl start > mysqldcat /var/log/mysqld.log grep "temporary password"mysql -uroot -pset > global validate_password_policy=0;set global validate_password_length=4;alter > user 'root'@'localhost' identified by '123456';mysql -uroot -p123456create > user 'root'@'%' identified by '123456';grant all privileges on *.* to > 'root'@'%' with grant option;flush privileges; > mkdir -p /usr/hive tar -zxvf /usr/hive/apache-hive-2.1.1-bin.tar.gz -C > /usr/hive/ > vim /etc/profile //for hiveexport > HIVE_HOME=/usr/hive/apache-hive-2.1.1-binexport > PATH=$PATH:$HIVE_HOME/binsource /etc/profile > vim hive-env.shcd $HIVE_HOME/conf && vim hive-env.shexport > HADOOP_HOME=/usr/hadoop/hadoop-2.7.3export > HIVE_CONF_DIR=/usr/hive/apache-hive-2.1.1-bin/confexport > HIVE_AUX_JARS_PATH=/usr/hive/apache-hive-2.1.1-bin/lib > cp $HIVE_HOME/lib/jline-2.12.jar $HADOOP_HOME/share/hadoop/yarn/lib/ > ##slave1 hive-servercd $HIVE_HOME/lib && wget or cp > mysql-connector-java-5.1.47-bin.jar > hive-site.xml (hive-server)hive.metastore.warehouse.dir > \{/user/hive_remote/warehouse}javax.jdo.option.ConnectionDriverName > \{com.mysql.jdbc.Driver}javax.jdo.option.ConnectionURL > \{jdbc:mysql://slave2:3306/hive?createDatabaseIfNotExist=true&useSSL=false}javax.jdo.option.ConnectionUserName > \{root}javax.jdo.option.ConnectionPassword \{123456} > hive-site.xml (hive client)hive.metastore.warehouse.dir > \{/user/hive_remote/warehouse}hive.metastore.local > \{false}hive.metastore.uris \{thrift://slave1:9083} > //opt hadoop fs -mkdir /tmp hadoop fs -mkdir -p /user/hive/warehousehadoop > fs -chmod g+w /tmp& /user/hive/warehouse//opt modify log locationvim > hive-log4j2.propertiesproperty.hive.log.dir=/home/hadoop > //opt hive-site.xmlhive.cli.print.current.db \{true}hive.cli.print.header > \{true} > beeline connect shoud config core-site.xmlhadoop.proxyuser.hadoop.hosts > \{*}..groups \{*}bin/beeline->!connect jdbc:hive2://centos01:10000or > bin/beeline -u jdbc:hive2://centos01:10000 -n hadoop > desc table/desc database/desc database extended testdb; > hive-serverschematool -dbType mysql -initSchemahive --service metastore > hive-clienthive \ncreate database hongyaa; > #install zookeepermkdir -p /usr/zookeeper && cd /usr/zookeepertar -zxvf > /usr/zookeeper/zookeeper-3.4.10.tar.gz -C /usr/zookeeper/vi /etc/profile > //for zookeeperexport > ZOOKEEPER_HOME=/usr/zookeeper/zookeeper-3.4.10PATH=$PATH:$ZOOKEEPER_HOME/binsource > /etc/profile > cd /usr/zookeeper/zookeeper-3.4.10/conf/ && mv zoo_sample.cfg zoo.cfg && > vimzoo.cfgtickTime=2000initLimit=10syncLimit=5dataDir=/usr/zookeeper/zookeeper-3.4.10/zkdataclientPort=2181dataLogDir=/usr/zookeeper/zookeeper-3.4.10/zkdatalogserver.1=master:2888:3888server.2=slave1:2888:3888server.3=slave2:2888:3888 > cd /usr/zookeeper/zookeeper-3.4.10 && mkdir zkdata zkdatalogcd > /usr/zookeeper/zookeeper-3.4.10/zkdata && echo "1" >> myid > zkServer.sh start(master,slave1,slave2)netstat -antp|grep 2181 > #install hbase//opttar -zxf hbase-1.2.6.1-bin.tar.gz -C /opt/modules/export > JAVA_HOME=/usr/java/jdk1.8.0_171export HBASE_MANAGES_ZK=true > vim hbase-site.xmlhbase.rootdir > \{hdfs://centos01:9000}hbase.cluster.distributed > \{true}hbase.zookeeper.quorum > \{centos01,centos02,centos03}hbase.zookeeper.property.dataDir > \{/opt/modules/hbase-1.2.6/zkData} > #install kafkatar -zxvf kafka_2.11-2.0.0.tgz -C /opt/modulesvim > config/server.propertiesbroker.id=1num.partitions=2default.replication.factor=1lisheners=PLAINTEXT://centos01:9092log.dirs=/opt/modules/kafka_2.11-2.0.0/kafka-logszookeeper.connect=centos01:2181,centos02:2181,centos03:2181scp > to centos02,centos03 and modify id > bin/kafka-server-start.sh -daemon > config/server.properties/kafka-server-start.sh -daemon > config/server.properties > create topicbin/kafka-topics.sh \--create \--zookeeper > centos01:2181,centos02:2181,centos03:2181 \--replication-factor 2 > \--partitions 2 \--topic topictest > bin/kafka-topics.sh \--list \--zookeeper centos01:2181 > bin/kafka-topics.sh \--describe \--zookeeper > centos01:2181//生产者bin/kafka-console-producer.sh \--broker-list > centos01:9092,centos02:9092,centos03:9092 \--topic > topictest//消费者bin/kafka-console-consumer.sh \--bootstrap-server > centos01:9092,centos02:9092,centos03:9092 \--topic topictest > java -Xms256m -Xmx512m -classpath .:./lib/* > com.cmcc.hsd.loader.ScannerV2p9Loader -t 6 -d 202008250000 > #install stormtar zxvf apache-storm-1.1.0.tar.gz -C /opt/modulesmv > apache-storm-1.1.0/ storm-1.1.0vim /etc/profileexport > STORM_HOME=/opt/modules/storm-1.1.0export PATH=$PATH:$STORM_HOME/bin > storm-env.shexport JAVA_HOME=/opt/softwares/jdk1.8.0_201export > STORM_CONF_DIR="/opt/modules/storm-1.1.0/conf"vim storm.yaml > storm.zookeeper.servers: - "centos01" - "centos02" - "centos03" > supervisor.slots.ports: - 6700 - 6701 - 6702 - 6703 > storm.zookeeper.port: 2181 storm.local.dir: "/opt/modules/storm-1.1.0/data" > scp -r to centos01,centos02,centos03 storm nimbus > /dev/null 2>&1 & > (master)storm ui > /dev/null 2>&1 & storm supervisor > /dev/null 2>&1 & > (slave) > jpsnimbus/Supervisor > #install sparktar zxvf spark-2.4.0-bin-hadoop2.7.tgz -C /opt/modulesvim > conf/slavescentos02centos03vim spark-env.shexport > JAVA_HOME=/opt/softwares/jdk1.8.0_201export SPARK_MASTER_IP=centos01export > SPARK_MASTER_PORT=7077 > export HADOOP_HOME=/opt/modules/hadoop-2.8.4export > HADOOP_CONF_DIR=$HADOOP_HOME/etc/haoop > scp -r to other node > spark ha delete SPARK_MASTER_IP add export > SPARK_DAEMON_JAVA_OPT="-Dspark.deploy.recoveryMode=ZOOKEEPER-Dspark.deploy.zookeeper.url=centos01:2181,centos02:2181,centos03:2181-Dspark.deploy.zookeepr.dir=/spark" > bin/spark-submit \--class org.apache.spark.examples.SparkPi \--master yarn > \--deploy-mode cluster > \/opt/modules/spark-2.4.0-bin-hadoop2.7/examples/jars/spark-examples_2.11-2.4.0.jar > bin/spark-shell \--master spark://centos02:7077 > bin/spark-submit \--class org.apache.spark.examples.SparkPi \--master yarn > \--deploy-mode cluster \--drive-memory 512m \--executor-memory 1g > \--executor-cores 2 > \/opt/modules/spark-2.4.0-bin-hadoop2.7/examples/jars/spark-examples_2.11-2.4.0.jar > yarn-site.xml ofen kill by yarnpmem-check-enablevmem-check-enable > #analysis data *****create database hive;use hive;create table bike > (duration int,startdate timestamp,enddate timestamp,type string) row format > delimitedfields terminated by ','; > LOAD DATA LOCAL INPATH './load.csv' INTO TABLE loan;insert overwrite local > directory '/home/hadoop/college001/'row format delimited fields terminated by > 't'select count(*) from bike;##about hiveDateType ,UDAF UDTFselect > from_unixtime cast(concat(startdate,':00') as timestamp),'yyyyMMdd000000') > from bike limit 10;select date_format('2019-10-07', 'yyyyMMdd000000')select > from_unixtime(unix_timestamp('2019-10-07', 'yyyy-MM-dd'), > 'yyyyMMdd000000')select unix_timestamp('2019-10-07 13:24:20','yyyy-MM-dd > HH:mm:ss') > JSON.stringify(option) > iconv -f gbk -t utf8 a.txt –o b.txt > mapreduce output -rw-r--r-- 2 hadoop supergroup 0 2021-10-26 07:06 > /output2/_SUCCESS-rw-r--r-- 2 hadoop supergroup 1306 2021-10-26 07:06 > /output2/part-r-00000 > #add node slave3yum install -y ntp && ntpdate mastervim /etc/hostsslave3scp > ~/.ssh/authorized_keys root@slave3:~/.ssh/yum install -y ntp && ntpdate > mastervim /etc/profile && source /etc/profileexport JAVA...export > HADOOP...vim $HADOOP_HOME/etc/hadoop/slaves add slave3 > $HADOOP_HOME/sbin/hadoop-daemon.sh start > datanode$HADOOP_HOME/sbin/yarn-daemon.sh start nodemanagerhdfs dfsadmin > -refreshNodessbin/start-balancer.shhdfs dfsadmin -report > echo "slave3" > excludes > hdfs-site.xml (master) > dfs.hosts.exclude \{/usr/hadoop/hadoop-2.7.3/etc/hadoop/excludes} > hdfs dfsadmin -refreshNodeshdfs dfsadmin -reporthadoop-daemon.sh stop > datanodeyarn-daemon.sh stop nodemanagerhdfs dfsadmin > -reportsbin/start-balancer.sh -- This message was sent by Atlassian Jira (v8.3.4#803005)