[
https://issues.apache.org/jira/browse/HIVE-17498?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16429741#comment-16429741
]
Igloo commented on HIVE-17498:
------------------------------
!image-2018-04-08-20-00-58-680.png!
> Does hive have mr-nativetask support refer to MAPREDUCE-2841
> ------------------------------------------------------------
>
> Key: HIVE-17498
> URL: https://issues.apache.org/jira/browse/HIVE-17498
> Project: Hive
> Issue Type: Bug
> Components: Hive
> Reporter: Feng Yuan
> Priority: Major
>
> I try to implement a HivePlatform extends
> org.apache.hadoop.mapred.nativetask.Platform.
> {code}
> /**
> * Licensed to the Apache Software Foundation (ASF) under one
> * or more contributor license agreements. See the NOTICE file
> * distributed with this work for additional information
> * regarding copyright ownership. The ASF licenses this file
> * to you under the Apache License, Version 2.0 (the
> * "License"); you may not use this file except in compliance
> * with the License. You may obtain a copy of the License at
> *
> * http://www.apache.org/licenses/LICENSE-2.0
> *
> * Unless required by applicable law or agreed to in writing, software
> * distributed under the License is distributed on an "AS IS" BASIS,
> * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> * See the License for the specific language governing permissions and
> * limitations under the License.
> */
> package org.apache.hadoop.mapred.nativetask;
> import org.apache.hadoop.hive.ql.io.HiveKey;
> import org.apache.hadoop.mapred.JobConf;
> import org.apache.hadoop.mapred.nativetask.serde.INativeSerializer;
> import org.apache.log4j.Logger;
> import java.io.DataInput;
> import java.io.DataOutput;
> import java.io.IOException;
> public class HivePlatform extends Platform {
> private static final Logger LOG = Logger.getLogger(HivePlatform.class);
> public HivePlatform() {
> }
> @Override
> public void init() throws IOException {
> registerKey("org.apache.hadoop.hive.ql.io.HiveKey",
> HiveKeySerializer.class);
> LOG.info("Hive platform inited");
> }
> @Override
> public String name() {
> return "Hive";
> }
> @Override
> public boolean support(String keyClassName, INativeSerializer serializer,
> JobConf job) {
> if (keyClassNames.contains(keyClassName) && serializer instanceof
> INativeComparable) {
> String nativeComparator = Constants.NATIVE_MAPOUT_KEY_COMPARATOR + "."
> + keyClassName;
> job.set(nativeComparator,
> "HivePlatform.HivePlatform::HiveKeyComparator");
> if (job.get(Constants.NATIVE_CLASS_LIBRARY_BUILDIN) == null) {
> job.set(Constants.NATIVE_CLASS_LIBRARY_BUILDIN,
> "HivePlatform=libnativetask.so");
> }
> return true;
> } else {
> return false;
> }
> }
> @Override
> public boolean define(Class comparatorClass) {
> return false;
> }
> public static class HiveKeySerializer implements INativeComparable,
> INativeSerializer<HiveKey> {
> public HiveKeySerializer() throws ClassNotFoundException,
> SecurityException, NoSuchMethodException {
> }
> @Override
> public int getLength(HiveKey w) throws IOException {
> return 4 + w.getLength();
> }
> @Override
> public void serialize(HiveKey w, DataOutput out) throws IOException {
> w.write(out);
> }
> @Override
> public void deserialize(DataInput in, int length, HiveKey w ) throws
> IOException {
> w.readFields(in);
> }
> }
> }
> {code}
> and throws exceptions:
> {code}
> Error: java.io.IOException: Initialization of all the collectors failed.
> Error in last collector was :Native output collector cannot be loaded; at
> org.apache.hadoop.mapred.MapTask.createSortingCollector(MapTask.java:415) at
> org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:442) at
> org.apache.hadoop.mapred.MapTask.run(MapTask.java:343) at
> org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164) at
> java.security.AccessController.doPrivileged(Native Method) at
> javax.security.auth.Subject.doAs(Subject.java:422) at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1700)
> at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158) Caused by:
> java.io.IOException: Native output collector cannot be loaded; at
> org.apache.hadoop.mapred.nativetask.NativeMapOutputCollectorDelegator.init(NativeMapOutputCollectorDelegator.java:165)
> at org.apache.hadoop.mapred.MapTask.createSortingCollector(MapTask.java:402)
> ... 7 more Caused by: java.io.IOException: /PartitionBucket.h:56:pool is
> NULL, or comparator is not set
> /usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(_ZN10NativeTask15HadoopExceptionC2ERKSs+0x76)
> [0x7ffdcbba6436]
> /usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(_ZN10NativeTask18MapOutputCollector4initEjjPFiPKcjS2_jEPNS_14ICombineRunnerE+0x36a)
> [0x7ffdcbb9ad6a]
> /usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(_ZN10NativeTask18MapOutputCollector9configureEPNS_6ConfigE+0x24a)
> [0x7ffdcbb9b37a]
> /usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(_ZN10NativeTask23MCollectorOutputHandler9configureEPNS_6ConfigE+0x80)
> [0x7ffdcbb91b40]
> /usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(_ZN10NativeTask12BatchHandler7onSetupEPNS_6ConfigEPcjS3_j+0xe9)
> [0x7ffdcbb90b29]
> /usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(Java_org_apache_hadoop_mapred_nativetask_NativeBatchProcessor_setupHandler+0x1fe)
> [0x7ffdcbb90e0e] [0x7ffde9017a34] at
> org.apache.hadoop.mapred.nativetask.NativeBatchProcessor.setupHandler(Native
> Method) at
> org.apache.hadoop.mapred.nativetask.NativeBatchProcessor.init(NativeBatchProcessor.java:122)
> at
> org.apache.hadoop.mapred.nativetask.NativeBatchProcessor.create(NativeBatchProcessor.java:91)
> at
> org.apache.hadoop.mapred.nativetask.handlers.NativeCollectorOnlyHandler.create(NativeCollectorOnlyHandler.java:85)
> at
> org.apache.hadoop.mapred.nativetask.NativeMapOutputCollectorDelegator.init(NativeMapOutputCollectorDelegator.java:161)
> ... 8 more
> {code}
> My environments are:
> hadoop 2.7.3 compile base on:
> {code}
> Linux x.x.x.x 2.6.32-220.el6.x86_64 #1 SMP Tue Dec 6 19:48:22 GMT 2011
> x86_64 x86_64 x86_64 GNU/Linux
> {code}
> compile command:
> {code}
> mvn package -DskipTests=true -Dmaven.javadoc.skip=true -Pdist,native
> -Dtar -Drequire.snappy -Dbundle.snappy -Dsnappy.lib=/usr/local/lib
> {code}
> hive running parameter:
> {code}
> set
> mapreduce.map.output.compress.codec=org.apache.hadoop.io.compress.SnappyCodec;
> set
> mapreduce.job.map.output.collector.class=org.apache.hadoop.mapred.nativetask.NativeMapOutputCollectorDelegator;
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
