This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/fluo-muchos.git
The following commit(s) were added to refs/heads/master by this push:
new 28991d6 Fix Muchos to work with Java 11 (#294)
28991d6 is described below
commit 28991d6d75b39de2181948b9f8bd5cafe5dea51f
Author: Arvind Shyamsundar <[email protected]>
AuthorDate: Tue Oct 29 17:29:52 2019 -0700
Fix Muchos to work with Java 11 (#294)
Fixes #266 by downloading the javax-activation-api artifiact to proxy
and then copying out to worker nodes.
---
ansible/roles/hadoop/tasks/main.yml | 6 ++++++
ansible/roles/proxy/tasks/download.yml | 10 ++++++++++
ansible/scripts/install_ansible.sh | 3 +++
lib/muchos/config.py | 22 ++++++++++++++++++++++
lib/muchos/existing.py | 1 +
5 files changed, 42 insertions(+)
diff --git a/ansible/roles/hadoop/tasks/main.yml
b/ansible/roles/hadoop/tasks/main.yml
index 24214e7..36c7a62 100644
--- a/ansible/roles/hadoop/tasks/main.yml
+++ b/ansible/roles/hadoop/tasks/main.yml
@@ -34,6 +34,12 @@
with_items:
- workers
when: hadoop_major_version == '3'
+
+# This is currently needed to run hadoop with Java 11 (see
https://github.com/apache/fluo-muchos/issues/266)
+- name: "Copy javax.activation-api (when Hadoop 3 and Java 11 are used)"
+ synchronize: src={{ user_home }}/mvn_dep/ dest={{ hadoop_home
}}/share/hadoop/common/lib/
+ when: hadoop_major_version == '3' and java_product_version == 11
+
- name: "copy spark yarn shuffle jar to hadoop lib"
command: cp {{ spark_home }}/yarn/spark-{{ spark_version }}-yarn-shuffle.jar
{{ hadoop_home }}/share/hadoop/yarn/lib/ creates={{ hadoop_home
}}/share/hadoop/yarn/lib/spark-{{ spark_version }}-yarn-shuffle.jar
when: "'spark' in groups"
diff --git a/ansible/roles/proxy/tasks/download.yml
b/ansible/roles/proxy/tasks/download.yml
index 48a3a7c..62e30b2 100644
--- a/ansible/roles/proxy/tasks/download.yml
+++ b/ansible/roles/proxy/tasks/download.yml
@@ -32,3 +32,13 @@
- { urlp: "{{ apache_mirror.stdout }}/hadoop/common/hadoop-{{
hadoop_version }}", fn: "{{ hadoop_tarball }}", sum: "{{ hadoop_sha256 }}" }
- { urlp: "{{ apache_mirror.stdout }}/maven/maven-3/{{ maven_version
}}/binaries", fn: "{{ maven_tarball }}", sum: "{{ maven_sha256 }}" }
- { urlp: "https://github.com/github/hub/releases/download/v{{ hub_version
}}", fn: "{{ hub_tarball }}", sum: "{{ hub_sha256 }}" }
+
+# This is currently needed to run hadoop with Java 11 (see
https://github.com/apache/fluo-muchos/issues/266)
+- name: "Download javax.activation-api for Hadoop 3 when Java 11 is used"
+ maven_artifact:
+ group_id: javax.activation
+ artifact_id: javax.activation-api
+ version: 1.2.0
+ dest: "{{ user_home }}/mvn_dep/"
+ mode: 0644
+ when: hadoop_major_version == '3' and java_product_version == 11
diff --git a/ansible/scripts/install_ansible.sh
b/ansible/scripts/install_ansible.sh
index 045d304..cc3ed62 100755
--- a/ansible/scripts/install_ansible.sh
+++ b/ansible/scripts/install_ansible.sh
@@ -41,3 +41,6 @@ if [ ! -h /etc/ansible/hosts ]; then
sudo rm -f hosts
sudo ln -s $base_dir/conf/hosts hosts
fi
+
+# install lxml as it is a dependency for the maven_artifact Ansible module
+sudo yum install -q -y python-lxml
diff --git a/lib/muchos/config.py b/lib/muchos/config.py
index 32bf4ad..c62c282 100644
--- a/lib/muchos/config.py
+++ b/lib/muchos/config.py
@@ -21,6 +21,7 @@ from .util import get_ephemeral_devices, get_arch
import os
import json
import glob
+from distutils.version import StrictVersion
SERVICES = ['zookeeper', 'namenode', 'resourcemanager', 'accumulomaster',
'mesosmaster', 'worker', 'fluo', 'fluo_yarn', 'metrics', 'spark', 'client',
'swarmmanager', 'journalnode', 'zkfc']
@@ -65,6 +66,11 @@ class DeployConfig(ConfigParser):
if not self.has_service(service):
exit("ERROR - Missing '{0}' service from [nodes]
section of muchos.props".format(service))
+ # validate if we are using Java 11 and fail if we are using
Accumulo 1.x
+ # See https://github.com/apache/accumulo/issues/958 for details
+ if self.java_product_version() >= 11 and
StrictVersion(self.version('accumulo')) <= StrictVersion("1.9.3"):
+ exit("ERROR - Java 11 is not supported with Accumulo version
'{0}'".format(self.version('accumulo')))
+
def verify_launch(self):
self.verify_instance_type(self.get('ec2', 'default_instance_type'))
self.verify_instance_type(self.get('ec2', 'worker_instance_type'))
@@ -188,6 +194,22 @@ class DeployConfig(ConfigParser):
def version(self, software_id):
return self.get('general', software_id + '_version')
+ def java_product_version(self):
+ java_version_map = {
+ "java-1.8.0-openjdk": 8,
+ "java-11-openjdk": 11,
+ "java-latest-openjdk": 13
+ }
+
+ # Given that a user might chose to install a specific JDK version
(where the version is suffixed to package name)
+ # it is safer to check if the configured Java version starts with one
of the above prefixes defined in the map.
+ configured_java_version = self.get("general", "java_package")
+ filtered_java_versions = {k:v for (k,v) in java_version_map.items() if
configured_java_version.startswith(k)}
+ if len(filtered_java_versions) != 1:
+ exit('ERROR - unknown or ambiguous Java version \'{0}\' specified
in properties'.format(configured_java_version))
+
+ return next(iter(filtered_java_versions.values()))
+
def checksum(self, software):
return self.checksum_ver(software, self.version(software))
diff --git a/lib/muchos/existing.py b/lib/muchos/existing.py
index b0edfe3..f1b3128 100644
--- a/lib/muchos/existing.py
+++ b/lib/muchos/existing.py
@@ -67,6 +67,7 @@ class ExistingCluster:
play_vars["force_format"] = config.force_format()
host_vars['worker_data_dirs'] = str(config.worker_data_dirs())
host_vars['default_data_dirs'] = str(config.default_data_dirs())
+ host_vars['java_product_version'] = str(config.java_product_version())
with open(join(config.deploy_path, "ansible/site.yml"), 'w') as
site_file:
print("- import_playbook: common.yml", file=site_file)
