This is an automated email from the ASF dual-hosted git repository.
arvindsh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fluo-muchos.git
The following commit(s) were added to refs/heads/main by this push:
new 1a23418 Avoid using Ansible module to download javax.activation-api
(#432)
1a23418 is described below
commit 1a2341803fd467d508e2e8fdf412aafea9d85648
Author: Arvind Shyamsundar <[email protected]>
AuthorDate: Wed Nov 1 11:24:21 2023 -0700
Avoid using Ansible module to download javax.activation-api (#432)
In #294 we had added support for Hadoop (<= 3.2.3) on Java 11 by
downloading and copying the javax.activation-api artifact using
Ansible's `maven_artifact` module. This module in turn needs `lxml`
which needed to be installed separately. The installation of `lxml`
ended up with variations (CentOS 7/8, Fedora) etc.
So this PR simplifies this by directly calling the Maven binary to
download the artifact on the proxy for later copying to the individual
Hadoop nodes.
---
ansible/hadoop.yml | 7 ++++
ansible/roles/proxy/tasks/download.yml | 70 ++++++++++++++++------------------
ansible/scripts/install_ansible.sh | 12 ------
3 files changed, 40 insertions(+), 49 deletions(-)
diff --git a/ansible/hadoop.yml b/ansible/hadoop.yml
index 37cdafa..2ed7ab7 100644
--- a/ansible/hadoop.yml
+++ b/ansible/hadoop.yml
@@ -15,6 +15,13 @@
# limitations under the License.
#
+- hosts: proxy
+ tasks:
+ # This is currently needed to run hadoop with Java 11 (see
https://github.com/apache/fluo-muchos/issues/266)
+ - name: "Download javax.activation-api for Hadoop 3 when Java 11 is used"
+ command: "{{ maven_home }}/bin/mvn dependency:copy
-Dartifact=javax.activation:javax.activation-api:1.2.0 -DoutputDirectory={{
user_home }}/mvn_dep/"
+ when: hadoop_major_version == '3' and java_product_version == 11
+
- hosts: all:!{{ azure_proxy_host }}
roles:
- hadoop
diff --git a/ansible/roles/proxy/tasks/download.yml
b/ansible/roles/proxy/tasks/download.yml
index d15c20d..b72a6ee 100644
--- a/ansible/roles/proxy/tasks/download.yml
+++ b/ansible/roles/proxy/tasks/download.yml
@@ -22,43 +22,39 @@
until: "'OK' in gresult.msg or 'file already exists' in gresult.msg"
retries: 3
with_items:
- - { urlp: "{{ apache_mirror.stdout }}/zookeeper/zookeeper-{{
zookeeper_version }}", fn: "{{ zookeeper_tarball }}", sum: "{{
zookeeper_checksum }}" }
- - { urlp: "{{ apache_mirror.stdout }}/hadoop/common/hadoop-{{
hadoop_version }}", fn: "{{ hadoop_tarball }}", sum: "{{ hadoop_checksum }}" }
- - { urlp: "{{ apache_mirror.stdout }}/maven/maven-3/{{ maven_version
}}/binaries", fn: "{{ maven_tarball }}", sum: "{{ maven_checksum }}" }
- - { urlp: "https://github.com/github/hub/releases/download/v{{ hub_version
}}", fn: "{{ hub_tarball }}", sum: "{{ hub_checksum }}" }
+ - urlp: "{{ apache_mirror.stdout }}/zookeeper/zookeeper-{{
zookeeper_version }}"
+ fn: "{{ zookeeper_tarball }}"
+ sum: "{{ zookeeper_checksum }}"
+ - urlp: "{{ apache_mirror.stdout }}/hadoop/common/hadoop-{{ hadoop_version
}}"
+ fn: "{{ hadoop_tarball }}"
+ sum: "{{ hadoop_checksum }}"
+ - urlp: "{{ apache_mirror.stdout }}/maven/maven-3/{{ maven_version
}}/binaries"
+ fn: "{{ maven_tarball }}"
+ sum: "{{ maven_checksum }}"
+ - urlp: "https://github.com/github/hub/releases/download/v{{ hub_version
}}"
+ fn: "{{ hub_tarball }}"
+ sum: "{{ hub_checksum }}"
when: "'snapshot' not in item.fn.lower()"
-# This is currently needed to run hadoop with Java 11 (see
https://github.com/apache/fluo-muchos/issues/266)
-- name: "Download javax.activation-api for Hadoop 3 when Java 11 is used"
- maven_artifact:
- group_id: javax.activation
- artifact_id: javax.activation-api
- repository_url: 'https://repo.maven.apache.org/maven2/'
- version: 1.2.0
- dest: "{{ user_home }}/mvn_dep/"
- mode: 0644
- when: hadoop_major_version == '3' and java_product_version == 11
-
-- block:
- - name: "download omsagent installer to proxy"
- shell: set -o pipefail && eval $(curl -s
https://raw.githubusercontent.com/Microsoft/OMS-Agent-for-Linux/master/installer/scripts/onboard_agent.sh
| grep -e 'GITHUB_RELEASE_X64=' -e 'BUNDLE_X64=') && wget
${GITHUB_RELEASE_X64}${BUNDLE_X64} -O {{ tarballs_dir }}/omsagent.x64.sh
- args:
- executable: bash
- register: result
- until: result is not failed
- retries: 3
- delay: 2
- when: az_oms_integration_needed
-
- - name: "check if Application Insights Agent jar was uploaded to proxy"
- stat: path={{ tarballs_dir }}/appinsights-agent.jar
- register: appinsights
- when: az_use_app_insights
- - name: "download Application Insights Agent jar to proxy"
- get_url:
url=https://github.com/microsoft/ApplicationInsights-Java/releases/download/{{
az_app_insights_version }}/applicationinsights-agent-{{ az_app_insights_version
}}.jar dest={{ tarballs_dir }}/appinsights-agent.jar force=no
- register: gresult
- until: "'OK' in gresult.msg or 'file already exists' in gresult.msg"
- retries: 3
- when: az_use_app_insights and not appinsights.stat.exists
+- name: "Tasks for Azure specific downloads"
when: cluster_type == 'azure'
-
+ block:
+ - name: "download omsagent installer to proxy"
+ shell: set -o pipefail && eval $(curl -s
https://raw.githubusercontent.com/Microsoft/OMS-Agent-for-Linux/master/installer/scripts/onboard_agent.sh
| grep -e 'GITHUB_RELEASE_X64=' -e 'BUNDLE_X64=') && wget
${GITHUB_RELEASE_X64}${BUNDLE_X64} -O {{ tarballs_dir }}/omsagent.x64.sh
+ args:
+ executable: bash
+ register: result
+ until: result is not failed
+ retries: 3
+ delay: 2
+ when: az_oms_integration_needed
+ - name: "check if Application Insights Agent jar was uploaded to proxy"
+ stat: path={{ tarballs_dir }}/appinsights-agent.jar
+ register: appinsights
+ when: az_use_app_insights
+ - name: "download Application Insights Agent jar to proxy"
+ get_url:
url=https://github.com/microsoft/ApplicationInsights-Java/releases/download/{{
az_app_insights_version }}/applicationinsights-agent-{{ az_app_insights_version
}}.jar dest={{ tarballs_dir }}/appinsights-agent.jar force=no
+ register: gresult
+ until: "'OK' in gresult.msg or 'file already exists' in gresult.msg"
+ retries: 3
+ when: az_use_app_insights and not appinsights.stat.exists
diff --git a/ansible/scripts/install_ansible.sh
b/ansible/scripts/install_ansible.sh
index 2286dfc..d2d4b9a 100755
--- a/ansible/scripts/install_ansible.sh
+++ b/ansible/scripts/install_ansible.sh
@@ -49,17 +49,5 @@ if [[ ! -h /etc/ansible/hosts ]]; then
sudo ln -s $base_dir/conf/hosts hosts
fi
-# install lxml as it is a dependency for the maven_artifact Ansible module
-centos_version=$(grep '^VERSION_ID=' /etc/os-release | cut -d'=' -f2 | tr -d
'"')
-if [[ $os_id = "centos" ]]; then
- if [[ $centos_version -eq 7 ]]; then
- sudo yum install -q -y python-lxml
- elif [[ $centos_version -eq 8 ]]; then
- sudo yum install -q -y python3-lxml
- fi
-elif [[ $os_id = "fedora" ]]; then
- sudo yum install -q -y python3-lxml
-fi
-
# install jq to ease JSON parsing on the proxy
sudo yum install -y jq
