This is an automated email from the ASF dual-hosted git repository.
mwalch pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/fluo-muchos.git
The following commit(s) were added to refs/heads/master by this push:
new 656a785 Improve config to handle different cluster types (#249)
656a785 is described below
commit 656a785e37eba2653cea96d8a6fd1158dd9c1b7d
Author: Mike Walch <[email protected]>
AuthorDate: Thu Jan 31 13:51:58 2019 -0500
Improve config to handle different cluster types (#249)
* Configuration object now returns correct config
based on cluster type
---
conf/muchos.props.example | 10 ++++---
lib/muchos/config.py | 67 ++++++++++++++++++++++++++++++++++++-----------
lib/muchos/existing.py | 29 ++++++--------------
lib/tests/test_config.py | 36 ++++++++++++++++++++++---
4 files changed, 98 insertions(+), 44 deletions(-)
diff --git a/conf/muchos.props.example b/conf/muchos.props.example
index c781238..899beef 100644
--- a/conf/muchos.props.example
+++ b/conf/muchos.props.example
@@ -80,10 +80,12 @@ shutdown_delay_minutes = 0
shutdown_behavior = stop
[existing]
-mount_root = /var/disk
-mounts = /var/disk01,/var/disk02,/var/disk03
-devices = /dev/hdb1,/dev/hdc1,/dev/hdd1
-metrics_drive_ids = var-disk01,var-disk02,var-disk03
+# Root of data dirs
+mount_root = /var/data
+# Data directories on all nodes
+data_dirs = /var/data1,/var/data2,/var/data3
+# Identifies drives for metrics
+metrics_drive_ids = var-data1,var-data2,var-data3
[performance]
# Automatically tune Accumulo, Yarn, and Fluo performance setting by selecting
or
diff --git a/lib/muchos/config.py b/lib/muchos/config.py
index d9a89bb..ace1532 100644
--- a/lib/muchos/config.py
+++ b/lib/muchos/config.py
@@ -36,7 +36,7 @@ class DeployConfig(ConfigParser):
self.cluster_name = cluster_name
self.sg_name = cluster_name + '-group'
self.ephemeral_root = 'ephemeral'
- self.mount_root = '/media/' + self.ephemeral_root
+ self.cluster_type = self.get('general', 'cluster_type')
self.metrics_drive_root = 'media-' + self.ephemeral_root
self.node_d = None
self.hosts = None
@@ -87,10 +87,11 @@ class DeployConfig(ConfigParser):
def node_type_map(self):
node_types = {}
- node_list = [('default', self.default_ephemeral_devices()), ('worker',
self.worker_ephemeral_devices())]
+ if self.get_cluster_type() == 'ec2':
+ node_list = [('default', self.default_ephemeral_devices()),
('worker', self.worker_ephemeral_devices())]
- for (ntype, devices) in node_list:
- node_types[ntype] = {'mounts': self.mounts(len(devices)),
'devices': devices}
+ for (ntype, devices) in node_list:
+ node_types[ntype] = {'mounts': self.mounts(len(devices)),
'devices': devices}
return node_types
def node_type(self, hostname):
@@ -101,26 +102,57 @@ class DeployConfig(ConfigParser):
def mounts(self, num_ephemeral):
mounts = []
for i in range(0, num_ephemeral):
- mounts.append(self.mount_root + str(i))
+ mounts.append(self.mount_root() + str(i))
return mounts
+ def mount_root(self):
+ if self.get_cluster_type() == 'ec2':
+ return '/media/' + self.ephemeral_root
+ elif self.get_cluster_type() == 'existing':
+ return self.get('existing', 'mount_root')
+
def fstype(self):
- retval = self.get('ec2', 'fstype')
- if not retval:
- return 'ext3'
+ retval = None
+ if self.get_cluster_type() == 'ec2':
+ retval = self.get('ec2', 'fstype')
+ if not retval:
+ return 'ext3'
return retval
def force_format(self):
- retval = self.get('ec2', 'force_format')
- if not retval:
- return 'no'
+ retval = 'no'
+ if self.get_cluster_type() == 'ec2':
+ retval = self.get('ec2', 'force_format')
+ if not retval:
+ return 'no'
return retval
+ def worker_data_dirs(self):
+ if self.get_cluster_type() == 'ec2':
+ return self.node_type_map()['worker']['mounts']
+ elif self.get_cluster_type() == 'existing':
+ return self.get('existing', 'data_dirs').split(",")
+
+ def default_data_dirs(self):
+ if self.get_cluster_type() == 'ec2':
+ return self.node_type_map()['default']['mounts']
+ elif self.get_cluster_type() == 'existing':
+ return self.get('existing', 'data_dirs').split(",")
+
def metrics_drive_ids(self):
- drive_ids = []
- for i in range(0, self.max_ephemeral()):
- drive_ids.append(self.metrics_drive_root + str(i))
- return drive_ids
+ if self.get_cluster_type() == 'ec2':
+ drive_ids = []
+ for i in range(0, self.max_ephemeral()):
+ drive_ids.append(self.metrics_drive_root + str(i))
+ return drive_ids
+ elif self.get_cluster_type() == 'existing':
+ return self.get("existing", "metrics_drive_ids").split(",")
+
+ def shutdown_delay_minutes(self):
+ retval = '0'
+ if self.get_cluster_type() == 'ec2':
+ retval = self.get("ec2", "shutdown_delay_minutes")
+ return retval
def version(self, software_id):
return self.get('general', software_id + '_version')
@@ -247,6 +279,11 @@ class DeployConfig(ConfigParser):
def get_public_ip(self, hostname):
return self.get_hosts()[hostname][1]
+ def get_cluster_type(self):
+ if self.cluster_type not in ('ec2', 'existing'):
+ exit('ERROR - Unknown cluster type' + self.cluster_type)
+ return self.cluster_type
+
def proxy_hostname(self):
return self.get('general', 'proxy_hostname')
diff --git a/lib/muchos/existing.py b/lib/muchos/existing.py
index 3fcd08e..6f79a4e 100644
--- a/lib/muchos/existing.py
+++ b/lib/muchos/existing.py
@@ -55,27 +55,14 @@ class ExistingCluster:
play_vars['hadoop_sha256'] = config.checksum('hadoop')
play_vars['spark_sha256'] = config.checksum('spark')
play_vars['zookeeper_sha256'] = config.checksum('zookeeper')
-
- cluster_type = host_vars.get('cluster_type', 'ec2')
- node_type_map = {}
- if cluster_type == 'ec2':
- node_type_map = config.node_type_map()
- play_vars["mount_root"] = config.mount_root
- play_vars["metrics_drive_ids"] = config.metrics_drive_ids()
- play_vars["fstype"] = config.fstype()
- play_vars["force_format"] = config.force_format()
- play_vars["shutdown_delay_minutes"] = config.get("ec2",
"shutdown_delay_minutes")
- if cluster_type == 'existing':
- play_vars["mount_root"] = config.get("existing", "mount_root")
- play_vars["metrics_drive_ids"] = config.get("existing",
"metrics_drives_ids").split(",")
- mounts = config.get("existing", "mounts").split(",")
- devices = config.get("existing", "devices").split(",")
- for node_type in 'default', 'worker':
- node_type_map[node_type] = {'mounts': mounts, 'devices':
devices}
-
- play_vars["node_type_map"] = node_type_map
- host_vars['worker_data_dirs'] = str(node_type_map['worker']['mounts'])
- host_vars['default_data_dirs'] =
str(node_type_map['default']['mounts'])
+ play_vars["shutdown_delay_minutes"] = config.shutdown_delay_minutes()
+ play_vars["metrics_drive_ids"] = config.metrics_drive_ids()
+ play_vars["mount_root"] = config.mount_root()
+ play_vars["node_type_map"] = config.node_type_map()
+ play_vars["fstype"] = config.fstype()
+ play_vars["force_format"] = config.force_format()
+ host_vars['worker_data_dirs'] = str(config.worker_data_dirs())
+ host_vars['default_data_dirs'] = str(config.default_data_dirs())
with open(join(config.deploy_path, "ansible/site.yml"), 'w') as
site_file:
print("- import_playbook: common.yml", file=site_file)
diff --git a/lib/tests/test_config.py b/lib/tests/test_config.py
index eed5a50..c69fc9a 100644
--- a/lib/tests/test_config.py
+++ b/lib/tests/test_config.py
@@ -18,7 +18,7 @@
from muchos.config import DeployConfig
-def test_defaults():
+def test_ec2_cluster():
c = DeployConfig("muchos", '../conf/muchos.props.example',
'../conf/hosts/example/example_cluster',
'../conf/checksums', 'mycluster')
assert c.checksum_ver('accumulo', '1.9.0') ==
'f68a6145029a9ea843b0305c90a7f5f0334d8a8ceeea94734267ec36421fe7fe'
@@ -27,6 +27,13 @@ def test_defaults():
assert c.get('ec2', 'worker_instance_type') == 'm5d.large'
assert c.get('ec2', 'aws_ami') == 'ami-9887c6e7'
assert c.max_ephemeral() == 1
+ assert c.mount_root() == '/media/ephemeral'
+ assert c.fstype() == 'ext3'
+ assert c.force_format() == 'no'
+ assert c.worker_data_dirs() == ['/media/ephemeral0']
+ assert c.default_data_dirs() == ['/media/ephemeral0']
+ assert c.metrics_drive_ids() == ['media-ephemeral0']
+ assert c.shutdown_delay_minutes() == '0'
assert c.mounts(2) == ['/media/ephemeral0', '/media/ephemeral1']
assert c.node_type_map() == {'default': {'mounts': ['/media/ephemeral0',
], 'devices': ['/dev/nvme1n1', ]},
'worker': {'mounts': ['/media/ephemeral0', ],
'devices': ['/dev/nvme1n1', ]}}
@@ -45,10 +52,13 @@ def test_defaults():
assert not c.has_service('fluo')
assert c.get_service_hostnames('worker') == ['worker1', 'worker2',
'worker3', 'worker4']
assert c.get_service_hostnames('zookeeper') == ['leader1']
- assert c.get_hosts() == {'leader2': ('10.0.0.1', None), 'leader1':
('10.0.0.0', '23.0.0.0'), 'worker1': ('10.0.0.2', None), 'worker3':
('10.0.0.4', None), 'worker2': ('10.0.0.3', None), 'worker4': ('10.0.0.5',
None)}
+ assert c.get_hosts() == {'leader2': ('10.0.0.1', None), 'leader1':
('10.0.0.0', '23.0.0.0'),
+ 'worker1': ('10.0.0.2', None), 'worker3':
('10.0.0.4', None),
+ 'worker2': ('10.0.0.3', None), 'worker4':
('10.0.0.5', None)}
assert c.get_public_ip('leader1') == '23.0.0.0'
assert c.get_private_ip('leader1') == '10.0.0.0'
assert c.cluster_name == 'mycluster'
+ assert c.get_cluster_type() == 'ec2'
assert c.version("accumulo").startswith('2.')
assert c.version("fluo").startswith('1.')
assert c.version("hadoop").startswith('3.')
@@ -61,8 +71,26 @@ def test_defaults():
assert c.get('general', 'cluster_user') == "centos"
assert c.get_non_proxy() == [('10.0.0.1', 'leader2'), ('10.0.0.2',
'worker1'), ('10.0.0.3', 'worker2'),
('10.0.0.4', 'worker3'), ('10.0.0.5',
'worker4')]
- assert c.get_host_services() == [('leader1', 'namenode resourcemanager
accumulomaster zookeeper'), ('leader2', 'metrics'),
- ('worker1', 'worker swarmmanager'), ('worker2', 'worker'),
('worker3', 'worker'), ('worker4', 'worker')]
+ assert c.get_host_services() == [('leader1', 'namenode resourcemanager
accumulomaster zookeeper'),
+ ('leader2', 'metrics'),
+ ('worker1', 'worker swarmmanager'),
('worker2', 'worker'), ('worker3', 'worker'),
+ ('worker4', 'worker')]
+
+
+def test_existing_cluster():
+ c = DeployConfig("muchos", '../conf/muchos.props.example',
'../conf/hosts/example/example_cluster',
+ '../conf/checksums', 'mycluster')
+ c.cluster_type = 'existing'
+ assert c.get_cluster_type() == 'existing'
+ assert c.node_type_map() == {}
+ assert c.mount_root() == '/var/data'
+ assert c.fstype() is None
+ assert c.force_format() == 'no'
+ assert c.worker_data_dirs() == ['/var/data1', '/var/data2', '/var/data3']
+ assert c.default_data_dirs() == ['/var/data1', '/var/data2', '/var/data3']
+ assert c.metrics_drive_ids() == ['var-data1', 'var-data2', 'var-data3']
+ assert c.shutdown_delay_minutes() == '0'
+
def test_case_sensitive():
c = DeployConfig("muchos", '../conf/muchos.props.example',
'../conf/hosts/example/example_cluster',
