This is an automated email from the ASF dual-hosted git repository.
mwalch pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/fluo-muchos.git
The following commit(s) were added to refs/heads/master by this push:
new 020b2f2 Users can configure Muchos to not download software (#250)
020b2f2 is described below
commit 020b2f2f57a8ecffee94ccdf00c0d69ee63edb77
Author: Mike Walch <[email protected]>
AuthorDate: Thu Jan 31 12:52:08 2019 -0500
Users can configure Muchos to not download software (#250)
* Option configured using 'download_software = False'
* Also use HADOOP_HOME in accumulo-env.sh for 2.0
---
ansible/accumulo.yml | 1 +
ansible/common.yml | 3 +++
ansible/fluo.yml | 1 +
ansible/roles/accumulo/templates/accumulo-env.sh | 2 +-
ansible/roles/proxy/tasks/{main.yml => download.yml} | 5 -----
ansible/roles/proxy/tasks/main.yml | 17 -----------------
ansible/spark.yml | 1 +
conf/muchos.props.example | 2 ++
lib/muchos/config.py | 1 +
lib/muchos/existing.py | 14 +++++---------
10 files changed, 15 insertions(+), 32 deletions(-)
diff --git a/ansible/accumulo.yml b/ansible/accumulo.yml
index b38695c..dca90eb 100644
--- a/ansible/accumulo.yml
+++ b/ansible/accumulo.yml
@@ -18,6 +18,7 @@
- hosts: proxy
tasks:
- import_tasks: roles/accumulo/tasks/download.yml
+ when: download_software
- hosts: all
roles:
- accumulo
diff --git a/ansible/common.yml b/ansible/common.yml
index 007cd03..95f7d85 100644
--- a/ansible/common.yml
+++ b/ansible/common.yml
@@ -22,6 +22,9 @@
- hosts: proxy
roles:
- proxy
+ tasks:
+ - import_tasks: roles/proxy/tasks/download.yml
+ when: download_software
- hosts: all
become: yes
roles:
diff --git a/ansible/fluo.yml b/ansible/fluo.yml
index e639d40..6a17e94 100644
--- a/ansible/fluo.yml
+++ b/ansible/fluo.yml
@@ -18,6 +18,7 @@
- hosts: proxy
tasks:
- import_tasks: roles/fluo/tasks/download.yml
+ when: download_software
- hosts: fluo
roles:
- fluo
diff --git a/ansible/roles/accumulo/templates/accumulo-env.sh
b/ansible/roles/accumulo/templates/accumulo-env.sh
index f6a5456..a6a1bc6 100755
--- a/ansible/roles/accumulo/templates/accumulo-env.sh
+++ b/ansible/roles/accumulo/templates/accumulo-env.sh
@@ -79,5 +79,5 @@ esac
export JAVA_OPTS
export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-1}
-export LD_LIBRARY_PATH="${HADOOP_PREFIX}/lib/native:${LD_LIBRARY_PATH}"
+export LD_LIBRARY_PATH="${HADOOP_HOME}/lib/native:${LD_LIBRARY_PATH}"
{% endif %}
diff --git a/ansible/roles/proxy/tasks/main.yml
b/ansible/roles/proxy/tasks/download.yml
similarity index 88%
copy from ansible/roles/proxy/tasks/main.yml
copy to ansible/roles/proxy/tasks/download.yml
index 711806e..5744070 100644
--- a/ansible/roles/proxy/tasks/main.yml
+++ b/ansible/roles/proxy/tasks/download.yml
@@ -15,11 +15,6 @@
# limitations under the License.
#
-- name: "ensure cluster user exists and generate ssh key"
- user: name={{ cluster_user }} generate_ssh_key=yes ssh_key_bits=4096
state=present
- become: yes
-- name: "create tarball directory on proxy"
- file: path={{ tarballs_dir }} state=directory
- name: "determine best apache mirror to use"
shell: curl -sk https://apache.org/mirrors.cgi?as_json | grep preferred |
cut -d \" -f 4
args:
diff --git a/ansible/roles/proxy/tasks/main.yml
b/ansible/roles/proxy/tasks/main.yml
index 711806e..21a7fa5 100644
--- a/ansible/roles/proxy/tasks/main.yml
+++ b/ansible/roles/proxy/tasks/main.yml
@@ -20,20 +20,3 @@
become: yes
- name: "create tarball directory on proxy"
file: path={{ tarballs_dir }} state=directory
-- name: "determine best apache mirror to use"
- shell: curl -sk https://apache.org/mirrors.cgi?as_json | grep preferred |
cut -d \" -f 4
- args:
- warn: no
- register: apache_mirror
- failed_when: "'http' not in apache_mirror.stdout"
- changed_when: False
-- name: "download common tarballs to proxy"
- get_url: url={{ item.urlp }}/{{ item.fn }} dest={{ tarballs_dir }}/{{
item.fn }} sha256sum={{ item.sum }} force=no
- register: gresult
- until: "'OK' in gresult.msg or 'file already exists' in gresult.msg"
- retries: 3
- with_items:
- - { urlp: "{{ apache_mirror.stdout }}/zookeeper/zookeeper-{{
zookeeper_version }}", fn: "{{ zookeeper_tarball }}", sum: "{{ zookeeper_sha256
}}" }
- - { urlp: "{{ apache_mirror.stdout }}/hadoop/common/hadoop-{{
hadoop_version }}", fn: "{{ hadoop_tarball }}", sum: "{{ hadoop_sha256 }}" }
- - { urlp: "{{ apache_mirror.stdout }}/maven/maven-3/{{ maven_version
}}/binaries", fn: "{{ maven_tarball }}", sum: "{{ maven_sha256 }}" }
- - { urlp: "https://github.com/github/hub/releases/download/v{{ hub_version
}}", fn: "{{ hub_tarball }}", sum: "{{ hub_sha256 }}" }
diff --git a/ansible/spark.yml b/ansible/spark.yml
index f28adf2..adf62b1 100644
--- a/ansible/spark.yml
+++ b/ansible/spark.yml
@@ -18,6 +18,7 @@
- hosts: proxy
tasks:
- import_tasks: roles/spark/tasks/download.yml
+ when: download_software
- hosts: all
roles:
- spark
diff --git a/conf/muchos.props.example b/conf/muchos.props.example
index d9825a2..c781238 100644
--- a/conf/muchos.props.example
+++ b/conf/muchos.props.example
@@ -39,6 +39,8 @@ spark_version = 2.3.2
fluo_version = 1.2.0
fluo_yarn_version = 1.0.0
accumulo_version = 2.0.0-alpha-1
+# Specifies if software should be downloaded. If 'False', tarballs of the
software above should be in conf/upload/
+download_software = True
[ec2]
# AWS machine image to use. The default below is for a CentOS 7 image (in
us-east-1).
diff --git a/lib/muchos/config.py b/lib/muchos/config.py
index 2fabce2..d9a89bb 100644
--- a/lib/muchos/config.py
+++ b/lib/muchos/config.py
@@ -301,6 +301,7 @@ HOST_VAR_DEFAULTS = {
'cluster_basedir': None,
'cluster_user': None,
'default_data_dirs': None,
+ 'download_software': None,
'fluo_home': '"{{ install_dir }}/fluo-{{ fluo_version }}"',
'fluo_tarball': 'fluo-{{ fluo_version }}-bin.tar.gz',
'fluo_version': None,
diff --git a/lib/muchos/existing.py b/lib/muchos/existing.py
index 46c48d3..3fcd08e 100644
--- a/lib/muchos/existing.py
+++ b/lib/muchos/existing.py
@@ -21,6 +21,7 @@ import subprocess
import time
from os.path import isfile, join
from sys import exit
+from os import listdir
from .config import HOST_VAR_DEFAULTS, PLAY_VAR_DEFAULTS
@@ -168,18 +169,13 @@ class ExistingCluster:
self.sync()
conf_upload = join(config.deploy_path, "conf/upload")
- accumulo_tarball = join(conf_upload,
"accumulo-{0}-bin.tar.gz".format(config.version("accumulo")))
- fluo_tarball = join(conf_upload,
"fluo-{0}-bin.tar.gz".format(config.version("fluo")))
- fluo_yarn_tarball = join(conf_upload,
"fluo-yarn-{0}-bin.tar.gz".format(config.version("fluo_yarn")))
basedir = config.get('general', 'cluster_basedir')
cluster_tarballs = "{0}/tarballs".format(basedir)
self.exec_on_proxy_verified("mkdir -p {0}".format(cluster_tarballs))
- if isfile(accumulo_tarball):
- self.send_to_proxy(accumulo_tarball, cluster_tarballs)
- if isfile(fluo_tarball) and config.has_service('fluo'):
- self.send_to_proxy(fluo_tarball, cluster_tarballs)
- if isfile(fluo_yarn_tarball) and config.has_service('fluo_yarn'):
- self.send_to_proxy(fluo_yarn_tarball, cluster_tarballs)
+ for f in listdir(conf_upload):
+ tarball_path = join(conf_upload, f)
+ if isfile(tarball_path) and tarball_path.endswith("gz"):
+ self.send_to_proxy(tarball_path, cluster_tarballs)
self.execute_playbook("site.yml")
