[
https://issues.apache.org/jira/browse/HIVE-17498?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Feng Yuan updated HIVE-17498:
-----------------------------
Description:
I try to implement a HivePlatform extends
org.apache.hadoop.mapred.nativetask.Platform.
{code}
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.mapred.nativetask;
import org.apache.hadoop.hive.ql.io.HiveKey;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.nativetask.serde.INativeSerializer;
import org.apache.log4j.Logger;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
public class HivePlatform extends Platform {
private static final Logger LOG = Logger.getLogger(HivePlatform.class);
public HivePlatform() {
}
@Override
public void init() throws IOException {
registerKey("org.apache.hadoop.hive.ql.io.HiveKey",
HiveKeySerializer.class);
LOG.info("Hive platform inited");
}
@Override
public String name() {
return "Hive";
}
@Override
public boolean support(String keyClassName, INativeSerializer serializer,
JobConf job) {
if (keyClassNames.contains(keyClassName) && serializer instanceof
INativeComparable) {
String nativeComparator = Constants.NATIVE_MAPOUT_KEY_COMPARATOR + "." +
keyClassName;
job.set(nativeComparator, "HivePlatform.HivePlatform::HiveKeyComparator");
if (job.get(Constants.NATIVE_CLASS_LIBRARY_BUILDIN) == null) {
job.set(Constants.NATIVE_CLASS_LIBRARY_BUILDIN,
"HivePlatform=libnativetask.so");
}
return true;
} else {
return false;
}
}
@Override
public boolean define(Class comparatorClass) {
return false;
}
public static class HiveKeySerializer implements INativeComparable,
INativeSerializer<HiveKey> {
public HiveKeySerializer() throws ClassNotFoundException,
SecurityException, NoSuchMethodException {
}
@Override
public int getLength(HiveKey w) throws IOException {
return 4 + w.getLength();
}
@Override
public void serialize(HiveKey w, DataOutput out) throws IOException {
w.write(out);
}
@Override
public void deserialize(DataInput in, int length, HiveKey w ) throws
IOException {
w.readFields(in);
}
}
}
{code}
and throws exceptions:
{code}
Error: java.io.IOException: Initialization of all the collectors failed. Error
in last collector was :Native output collector cannot be loaded; at
org.apache.hadoop.mapred.MapTask.createSortingCollector(MapTask.java:415) at
org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:442) at
org.apache.hadoop.mapred.MapTask.run(MapTask.java:343) at
org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164) at
java.security.AccessController.doPrivileged(Native Method) at
javax.security.auth.Subject.doAs(Subject.java:422) at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1700)
at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158) Caused by:
java.io.IOException: Native output collector cannot be loaded; at
org.apache.hadoop.mapred.nativetask.NativeMapOutputCollectorDelegator.init(NativeMapOutputCollectorDelegator.java:165)
at org.apache.hadoop.mapred.MapTask.createSortingCollector(MapTask.java:402)
... 7 more Caused by: java.io.IOException: /PartitionBucket.h:56:pool is NULL,
or comparator is not set
/usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(_ZN10NativeTask15HadoopExceptionC2ERKSs+0x76)
[0x7ffdcbba6436]
/usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(_ZN10NativeTask18MapOutputCollector4initEjjPFiPKcjS2_jEPNS_14ICombineRunnerE+0x36a)
[0x7ffdcbb9ad6a]
/usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(_ZN10NativeTask18MapOutputCollector9configureEPNS_6ConfigE+0x24a)
[0x7ffdcbb9b37a]
/usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(_ZN10NativeTask23MCollectorOutputHandler9configureEPNS_6ConfigE+0x80)
[0x7ffdcbb91b40]
/usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(_ZN10NativeTask12BatchHandler7onSetupEPNS_6ConfigEPcjS3_j+0xe9)
[0x7ffdcbb90b29]
/usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(Java_org_apache_hadoop_mapred_nativetask_NativeBatchProcessor_setupHandler+0x1fe)
[0x7ffdcbb90e0e] [0x7ffde9017a34] at
org.apache.hadoop.mapred.nativetask.NativeBatchProcessor.setupHandler(Native
Method) at
org.apache.hadoop.mapred.nativetask.NativeBatchProcessor.init(NativeBatchProcessor.java:122)
at
org.apache.hadoop.mapred.nativetask.NativeBatchProcessor.create(NativeBatchProcessor.java:91)
at
org.apache.hadoop.mapred.nativetask.handlers.NativeCollectorOnlyHandler.create(NativeCollectorOnlyHandler.java:85)
at
org.apache.hadoop.mapred.nativetask.NativeMapOutputCollectorDelegator.init(NativeMapOutputCollectorDelegator.java:161)
... 8 more
{code}
My environments are:
hadoop 2.7.3 compile base on:
{code}
Linux x.x.x.x 2.6.32-220.el6.x86_64 #1 SMP Tue Dec 6 19:48:22 GMT 2011
x86_64 x86_64 x86_64 GNU/Linux
{code}
compile command:
{code}
mvn package -DskipTests=true -Dmaven.javadoc.skip=true -Pdist,native -Dtar
-Drequire.snappy -Dbundle.snappy -Dsnappy.lib=/usr/local/lib
{code}
hive running parameter:
{code}
set
mapreduce.map.output.compress.codec=org.apache.hadoop.io.compress.SnappyCodec;
set
mapreduce.job.map.output.collector.class=org.apache.hadoop.mapred.nativetask.NativeMapOutputCollectorDelegator;
{code}
was:
I try to implement a HivePlatform extends
org.apache.hadoop.mapred.nativetask.Platform.
{code}
//
// Source code recreated from a .class file by IntelliJ IDEA
// (powered by Fernflower decompiler)
//
package org.apache.hadoop.mapred.nativetask;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.nativetask.serde.INativeSerializer;
import org.apache.hadoop.mapred.nativetask.serde.NativeSerialization;
@Public
@Evolving
public abstract class Platform {
private final NativeSerialization serialization =
NativeSerialization.getInstance();
protected Set<String> keyClassNames = new HashSet();
public Platform() {
}
public abstract void init() throws IOException;
public abstract String name();
protected void registerKey(String keyClassName, Class<?> key) throws
IOException {
this.serialization.register(keyClassName, key);
this.keyClassNames.add(keyClassName);
}
protected abstract boolean support(String var1, INativeSerializer<?> var2,
JobConf var3);
protected abstract boolean define(Class<?> var1);
}
{code}
> Does hive have mr-nativetask support refer to MAPREDUCE-2841
> ------------------------------------------------------------
>
> Key: HIVE-17498
> URL: https://issues.apache.org/jira/browse/HIVE-17498
> Project: Hive
> Issue Type: Bug
> Components: Hive
> Reporter: Feng Yuan
>
> I try to implement a HivePlatform extends
> org.apache.hadoop.mapred.nativetask.Platform.
> {code}
> /**
> * Licensed to the Apache Software Foundation (ASF) under one
> * or more contributor license agreements. See the NOTICE file
> * distributed with this work for additional information
> * regarding copyright ownership. The ASF licenses this file
> * to you under the Apache License, Version 2.0 (the
> * "License"); you may not use this file except in compliance
> * with the License. You may obtain a copy of the License at
> *
> * http://www.apache.org/licenses/LICENSE-2.0
> *
> * Unless required by applicable law or agreed to in writing, software
> * distributed under the License is distributed on an "AS IS" BASIS,
> * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> * See the License for the specific language governing permissions and
> * limitations under the License.
> */
> package org.apache.hadoop.mapred.nativetask;
> import org.apache.hadoop.hive.ql.io.HiveKey;
> import org.apache.hadoop.mapred.JobConf;
> import org.apache.hadoop.mapred.nativetask.serde.INativeSerializer;
> import org.apache.log4j.Logger;
> import java.io.DataInput;
> import java.io.DataOutput;
> import java.io.IOException;
> public class HivePlatform extends Platform {
> private static final Logger LOG = Logger.getLogger(HivePlatform.class);
> public HivePlatform() {
> }
> @Override
> public void init() throws IOException {
> registerKey("org.apache.hadoop.hive.ql.io.HiveKey",
> HiveKeySerializer.class);
> LOG.info("Hive platform inited");
> }
> @Override
> public String name() {
> return "Hive";
> }
> @Override
> public boolean support(String keyClassName, INativeSerializer serializer,
> JobConf job) {
> if (keyClassNames.contains(keyClassName) && serializer instanceof
> INativeComparable) {
> String nativeComparator = Constants.NATIVE_MAPOUT_KEY_COMPARATOR + "."
> + keyClassName;
> job.set(nativeComparator,
> "HivePlatform.HivePlatform::HiveKeyComparator");
> if (job.get(Constants.NATIVE_CLASS_LIBRARY_BUILDIN) == null) {
> job.set(Constants.NATIVE_CLASS_LIBRARY_BUILDIN,
> "HivePlatform=libnativetask.so");
> }
> return true;
> } else {
> return false;
> }
> }
> @Override
> public boolean define(Class comparatorClass) {
> return false;
> }
> public static class HiveKeySerializer implements INativeComparable,
> INativeSerializer<HiveKey> {
> public HiveKeySerializer() throws ClassNotFoundException,
> SecurityException, NoSuchMethodException {
> }
> @Override
> public int getLength(HiveKey w) throws IOException {
> return 4 + w.getLength();
> }
> @Override
> public void serialize(HiveKey w, DataOutput out) throws IOException {
> w.write(out);
> }
> @Override
> public void deserialize(DataInput in, int length, HiveKey w ) throws
> IOException {
> w.readFields(in);
> }
> }
> }
> {code}
> and throws exceptions:
> {code}
> Error: java.io.IOException: Initialization of all the collectors failed.
> Error in last collector was :Native output collector cannot be loaded; at
> org.apache.hadoop.mapred.MapTask.createSortingCollector(MapTask.java:415) at
> org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:442) at
> org.apache.hadoop.mapred.MapTask.run(MapTask.java:343) at
> org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164) at
> java.security.AccessController.doPrivileged(Native Method) at
> javax.security.auth.Subject.doAs(Subject.java:422) at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1700)
> at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158) Caused by:
> java.io.IOException: Native output collector cannot be loaded; at
> org.apache.hadoop.mapred.nativetask.NativeMapOutputCollectorDelegator.init(NativeMapOutputCollectorDelegator.java:165)
> at org.apache.hadoop.mapred.MapTask.createSortingCollector(MapTask.java:402)
> ... 7 more Caused by: java.io.IOException: /PartitionBucket.h:56:pool is
> NULL, or comparator is not set
> /usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(_ZN10NativeTask15HadoopExceptionC2ERKSs+0x76)
> [0x7ffdcbba6436]
> /usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(_ZN10NativeTask18MapOutputCollector4initEjjPFiPKcjS2_jEPNS_14ICombineRunnerE+0x36a)
> [0x7ffdcbb9ad6a]
> /usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(_ZN10NativeTask18MapOutputCollector9configureEPNS_6ConfigE+0x24a)
> [0x7ffdcbb9b37a]
> /usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(_ZN10NativeTask23MCollectorOutputHandler9configureEPNS_6ConfigE+0x80)
> [0x7ffdcbb91b40]
> /usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(_ZN10NativeTask12BatchHandler7onSetupEPNS_6ConfigEPcjS3_j+0xe9)
> [0x7ffdcbb90b29]
> /usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(Java_org_apache_hadoop_mapred_nativetask_NativeBatchProcessor_setupHandler+0x1fe)
> [0x7ffdcbb90e0e] [0x7ffde9017a34] at
> org.apache.hadoop.mapred.nativetask.NativeBatchProcessor.setupHandler(Native
> Method) at
> org.apache.hadoop.mapred.nativetask.NativeBatchProcessor.init(NativeBatchProcessor.java:122)
> at
> org.apache.hadoop.mapred.nativetask.NativeBatchProcessor.create(NativeBatchProcessor.java:91)
> at
> org.apache.hadoop.mapred.nativetask.handlers.NativeCollectorOnlyHandler.create(NativeCollectorOnlyHandler.java:85)
> at
> org.apache.hadoop.mapred.nativetask.NativeMapOutputCollectorDelegator.init(NativeMapOutputCollectorDelegator.java:161)
> ... 8 more
> {code}
> My environments are:
> hadoop 2.7.3 compile base on:
> {code}
> Linux x.x.x.x 2.6.32-220.el6.x86_64 #1 SMP Tue Dec 6 19:48:22 GMT 2011
> x86_64 x86_64 x86_64 GNU/Linux
> {code}
> compile command:
> {code}
> mvn package -DskipTests=true -Dmaven.javadoc.skip=true -Pdist,native
> -Dtar -Drequire.snappy -Dbundle.snappy -Dsnappy.lib=/usr/local/lib
> {code}
> hive running parameter:
> {code}
> set
> mapreduce.map.output.compress.codec=org.apache.hadoop.io.compress.SnappyCodec;
> set
> mapreduce.job.map.output.collector.class=org.apache.hadoop.mapred.nativetask.NativeMapOutputCollectorDelegator;
> {code}
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
