Repository: libcloud Updated Branches: refs/heads/trunk 4158507b0 -> a38ade584
LIBCLOUD-578: GCE adding Service Accounts to create_node Closes #372 Signed-off-by: Tomaz Muraus <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/a38ade58 Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/a38ade58 Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/a38ade58 Branch: refs/heads/trunk Commit: a38ade58404f155c5de82a2a9512809856f63f17 Parents: 4158507 Author: Eric Johnson <[email protected]> Authored: Fri Oct 10 15:52:02 2014 +0000 Committer: Tomaz Muraus <[email protected]> Committed: Fri Oct 17 15:20:41 2014 +0800 ---------------------------------------------------------------------- CHANGES.rst | 5 + docs/compute/drivers/gce.rst | 14 +++ .../compute/gce/gce_service_account_scopes.py | 31 +++++ libcloud/compute/drivers/gce.py | 124 +++++++++++++++++-- libcloud/test/compute/test_gce.py | 26 ++++ 5 files changed, 193 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/a38ade58/CHANGES.rst ---------------------------------------------------------------------- diff --git a/CHANGES.rst b/CHANGES.rst index a23f4eb..e7b9542 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -151,6 +151,11 @@ Compute (GITHUB-370) [Nirmal Ranganathan] +- Add support for service scopes to the ``create_node`` method in the GCE + driver. + (LIBCLOUD-578, GITHUB-373) + [Eric Johnson] + Storage ~~~~~~~ http://git-wip-us.apache.org/repos/asf/libcloud/blob/a38ade58/docs/compute/drivers/gce.rst ---------------------------------------------------------------------- diff --git a/docs/compute/drivers/gce.rst b/docs/compute/drivers/gce.rst index a6b6b71..6e91160 100644 --- a/docs/compute/drivers/gce.rst +++ b/docs/compute/drivers/gce.rst @@ -15,6 +15,7 @@ Google Compute Engine features: * High-performance virtual machines * Minute-level billing (10-minute minimum) * Fast VM provisioning +* Persistent block storage (SSD and standard) * Native Load Balancing Connecting to Google Compute Engine @@ -68,6 +69,14 @@ To set up Installed Account authentication: 7. You will also need your "Project ID" which can be found by clicking on the "Overview" link on the left sidebar. +Accessing Google Cloud services from your Libcloud nodes +-------------------------------------------------------- +In order for nodes created with libcloud to be able to access or manage other +Google Cloud Platform services, you will need to specify a list of Service +Account Scopes. By default libcloud will create nodes that only allow +read-only access to Google Cloud Storage. A few of the examples below +illustrate how to use Service Account Scopes. + Examples -------- @@ -89,6 +98,11 @@ https://github.com/apache/libcloud/blob/trunk/demos/gce_demo.py .. literalinclude:: /examples/compute/gce/gce_datacenter.py +4. Specifying Service Account Scopes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. literalinclude:: /examples/compute/gce/gce_service_account_scopes.py + API Docs -------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/a38ade58/docs/examples/compute/gce/gce_service_account_scopes.py ---------------------------------------------------------------------- diff --git a/docs/examples/compute/gce/gce_service_account_scopes.py b/docs/examples/compute/gce/gce_service_account_scopes.py new file mode 100644 index 0000000..f5f9be9 --- /dev/null +++ b/docs/examples/compute/gce/gce_service_account_scopes.py @@ -0,0 +1,31 @@ +# See previous examples for connecting and creating the driver +# ... +driver = None + +# Define common example attributes +s = 'n1-standard-1' +i = 'debian-7' +z = 'us-central1-a' + +# Service Account Scopes require a list of dictionaries. Each dictionary +# can have an optional 'email' address specifying the Service Account +# address, and list of 'scopes'. The default Service Account Scopes for +# new nodes will effectively use: + +sa_scopes = [ + { + 'email': 'default', + 'scopes': ['storage-ro'] + } +] + +# The expected scenario will likely use the default Service Account email +# address, but allow users to override the default list of scopes. +# For example, create a new node with full access to Google Cloud Storage +# and Google Compute Engine: +sa_scopes = [{'scopes': ['compute', 'storage-full']}] +node_1 = driver.create_node("n1", s, i, z, ex_service_accounts=sa_scopes) + +# See Google's documentation for Accessing other Google Cloud services from +# your Google Compute Engine instances at, +# https://cloud.google.com/compute/docs/authentication http://git-wip-us.apache.org/repos/asf/libcloud/blob/a38ade58/libcloud/compute/drivers/gce.py ---------------------------------------------------------------------- diff --git a/libcloud/compute/drivers/gce.py b/libcloud/compute/drivers/gce.py index bbcfbb8..856c334 100644 --- a/libcloud/compute/drivers/gce.py +++ b/libcloud/compute/drivers/gce.py @@ -534,6 +534,22 @@ class GCENodeDriver(NodeDriver): "TERMINATED": NodeState.TERMINATED } + AUTH_URL = "https://www.googleapis.com/auth/"; + SA_SCOPES_MAP = { + # list derived from 'gcloud compute instances create --help' + "bigquery": "bigquery", + "compute-ro": "compute.readonly", + "compute-rw": "compute", + "datastore": "datastore", + "sql": "sqlservice", + "sql-admin": "sqlservice.admin", + "storage-full": "devstorage.full_control", + "storage-ro": "devstorage.read_only", + "storage-rw": "devstorage.read_write", + "taskqueue": "taskqueue", + "userinfo-email": "userinfo.email" + } + def __init__(self, user_id, key, datacenter=None, project=None, auth_type=None, scopes=None, **kwargs): """ @@ -1222,7 +1238,7 @@ class GCENodeDriver(NodeDriver): ex_network='default', ex_tags=None, ex_metadata=None, ex_boot_disk=None, use_existing_disk=True, external_ip='ephemeral', ex_disk_type='pd-standard', - ex_disk_auto_delete=True): + ex_disk_auto_delete=True, ex_service_accounts=None): """ Create a new node and return a node object for the node. @@ -1273,6 +1289,20 @@ class GCENodeDriver(NodeDriver): True by default. :type ex_disk_auto_delete: ``bool`` + :keyword ex_service_accounts: Specify a list of serviceAccounts when + creating the instance. The format is a + list of dictionaries containing email + and list of scopes, e.g. + [{'email':'default', + 'scopes':['compute', ...]}, ...] + Scopes can either be full URLs or short + names. If not provided, use the + 'default' service account email and a + scope of 'devstorage.read_only'. Also + accepts the aliases defined in + 'gcloud cmopute'. + :type ex_service_accounts: ``list`` + :return: A Node object for the new node. :rtype: :class:`Node` """ @@ -1315,7 +1345,8 @@ class GCENodeDriver(NodeDriver): ex_tags, ex_metadata, ex_boot_disk, external_ip, ex_disk_type, - ex_disk_auto_delete) + ex_disk_auto_delete, + ex_service_accounts) self.connection.async_request(request, method='POST', data=node_data) return self.ex_get_node(name, location.name) @@ -1327,6 +1358,7 @@ class GCENodeDriver(NodeDriver): poll_interval=2, external_ip='ephemeral', ex_disk_type='pd-standard', ex_auto_disk_delete=True, + ex_service_accounts=None, timeout=DEFAULT_TASK_COMPLETION_TIMEOUT): """ Create multiple nodes and return a list of Node objects. @@ -1392,6 +1424,20 @@ class GCENodeDriver(NodeDriver): True by default. :type ex_disk_auto_delete: ``bool`` + :keyword ex_service_accounts: Specify a list of serviceAccounts when + creating the instance. The format is a + list of dictionaries containing email + and list of scopes, e.g. + [{'email':'default', + 'scopes':['compute', ...]}, ...] + Scopes can either be full URLs or short + names. If not provided, use the + 'default' service account email and a + scope of 'devstorage.read_only'. Also + accepts the aliases defined in + 'gcloud cmopute'. + :type ex_service_accounts: ``list`` + :keyword timeout: The number of seconds to wait for all nodes to be created before timing out. :type timeout: ``int`` @@ -1418,7 +1464,8 @@ class GCENodeDriver(NodeDriver): 'ignore_errors': ignore_errors, 'use_existing_disk': use_existing_disk, 'external_ip': external_ip, - 'ex_disk_type': ex_disk_type} + 'ex_disk_type': ex_disk_type, + 'ex_service_accounts': ex_service_accounts} # List for holding the status information for disk/node creation. status_list = [] @@ -1930,7 +1977,8 @@ class GCENodeDriver(NodeDriver): return success def deploy_node(self, name, size, image, script, location=None, - ex_network='default', ex_tags=None): + ex_network='default', ex_tags=None, + ex_service_accounts=None): """ Create a new node and run a script on start-up. @@ -1956,6 +2004,20 @@ class GCENodeDriver(NodeDriver): :keyword ex_tags: A list of tags to associate with the node. :type ex_tags: ``list`` of ``str`` or ``None`` + :keyword ex_service_accounts: Specify a list of serviceAccounts when + creating the instance. The format is a + list of dictionaries containing email + and list of scopes, e.g. + [{'email':'default', + 'scopes':['compute', ...]}, ...] + Scopes can either be full URLs or short + names. If not provided, use the + 'default' service account email and a + scope of 'devstorage.read_only'. Also + accepts the aliases defined in + 'gcloud cmopute'. + :type ex_service_accounts: ``list`` + :return: A Node object for the new node. :rtype: :class:`Node` """ @@ -1966,7 +2028,8 @@ class GCENodeDriver(NodeDriver): return self.create_node(name, size, image, location=location, ex_network=ex_network, ex_tags=ex_tags, - ex_metadata=metadata) + ex_metadata=metadata, + ex_service_accounts=ex_service_accounts) def attach_volume(self, node, volume, device=None, ex_mode=None, ex_boot=False): @@ -2863,7 +2926,7 @@ class GCENodeDriver(NodeDriver): def _create_node_req(self, name, size, image, location, network, tags=None, metadata=None, boot_disk=None, external_ip='ephemeral', ex_disk_type='pd-standard', - ex_disk_auto_delete=True): + ex_disk_auto_delete=True, ex_service_accounts=None): """ Returns a request and body to create a new node. This is a helper method to support both :class:`create_node` and @@ -2910,6 +2973,20 @@ class GCENodeDriver(NodeDriver): True by default. :type ex_disk_auto_delete: ``bool`` + :keyword ex_service_accounts: Specify a list of serviceAccounts when + creating the instance. The format is a + list of dictionaries containing email + and list of scopes, e.g. + [{'email':'default', + 'scopes':['compute', ...]}, ...] + Scopes can either be full URLs or short + names. If not provided, use the + 'default' service account email and a + scope of 'devstorage.read_only'. Also + accepts the aliases defined in + 'gcloud cmopute'. + :type ex_service_accounts: ``list`` + :return: A tuple containing a request string and a node_data dict. :rtype: ``tuple`` of ``str`` and ``dict`` """ @@ -2921,6 +2998,38 @@ class GCENodeDriver(NodeDriver): if metadata: node_data['metadata'] = metadata + # by default, new instances will match the same serviceAccount and + # scope set in the Developers Console and Cloud SDK + if not ex_service_accounts: + set_scopes = [{ + 'email': 'default', + 'scopes': [self.AUTH_URL + 'devstorage.read_only'] + }] + elif not isinstance(ex_service_accounts, list): + raise ValueError("ex_service_accounts field is not a list.") + else: + set_scopes = [] + for sa in ex_service_accounts: + if not isinstance(sa, dict): + raise ValueError("ex_service_accounts needs to be a list " + "of dicts, got: '%s - %s'" % ( + str(type(sa)), str(sa))) + if 'email' not in sa: + sa['email'] = 'default' + if 'scopes' not in sa: + sa['scopes'] = [self.AUTH_URL + 'devstorage.read_only'] + ps = [] + for scope in sa['scopes']: + if scope.startswith(self.AUTH_URL): + ps.append(scope) + elif scope in self.SA_SCOPES_MAP: + ps.append(self.AUTH_URL + self.SA_SCOPES_MAP[scope]) + else: + ps.append(self.AUTH_URL + scope) + sa['scopes'] = ps + set_scopes.append(sa) + node_data['serviceAccounts'] = set_scopes + if boot_disk: disks = [{'kind': 'compute#attachedDisk', 'boot': True, @@ -3040,7 +3149,8 @@ class GCENodeDriver(NodeDriver): status['name'], node_attrs['size'], node_attrs['image'], node_attrs['location'], node_attrs['network'], node_attrs['tags'], node_attrs['metadata'], boot_disk=status['disk'], - external_ip=node_attrs['external_ip']) + external_ip=node_attrs['external_ip'], + ex_service_accounts=node_attrs['ex_service_accounts']) try: node_res = self.connection.request( request, method='POST', data=node_data).object http://git-wip-us.apache.org/repos/asf/libcloud/blob/a38ade58/libcloud/test/compute/test_gce.py ---------------------------------------------------------------------- diff --git a/libcloud/test/compute/test_gce.py b/libcloud/test/compute/test_gce.py index 730ef27..97bb61f 100644 --- a/libcloud/test/compute/test_gce.py +++ b/libcloud/test/compute/test_gce.py @@ -302,6 +302,11 @@ class GCENodeDriverTest(LibcloudTestCase, TestCaseMixin): self.assertEqual(node_data['tags']['items'][0], 'libcloud') self.assertEqual(node_data['name'], 'lcnode') self.assertTrue(node_data['disks'][0]['boot']) + self.assertIsInstance(node_data['serviceAccounts'], list) + self.assertIsInstance(node_data['serviceAccounts'][0], dict) + self.assertTrue(node_data['serviceAccounts'][0]['email'], 'default') + self.assertIsInstance(node_data['serviceAccounts'][0]['scopes'], list) + self.assertTrue(len(node_data['serviceAccounts'][0]['scopes']), 1) def test_create_node(self): node_name = 'node-name' @@ -311,6 +316,27 @@ class GCENodeDriverTest(LibcloudTestCase, TestCaseMixin): self.assertTrue(isinstance(node, Node)) self.assertEqual(node.name, node_name) + def test_create_node_req_with_serviceaccounts(self): + image = self.driver.ex_get_image('debian-7') + size = self.driver.ex_get_size('n1-standard-1') + location = self.driver.zone + network = self.driver.ex_get_network('default') + # ex_service_accounts with specific scopes, default 'email' + ex_sa = [{'scopes': ['compute-ro', 'pubsub', 'storage-ro']}] + node_request, node_data = self.driver._create_node_req('lcnode', size, + image, location, + network, + ex_service_accounts=ex_sa) + self.assertIsInstance(node_data['serviceAccounts'], list) + self.assertIsInstance(node_data['serviceAccounts'][0], dict) + self.assertTrue(node_data['serviceAccounts'][0]['email'], 'default') + self.assertIsInstance(node_data['serviceAccounts'][0]['scopes'], list) + self.assertTrue(len(node_data['serviceAccounts'][0]['scopes']), 3) + self.assertTrue('https://www.googleapis.com/auth/devstorage.read_only' + in node_data['serviceAccounts'][0]['scopes']) + self.assertTrue('https://www.googleapis.com/auth/compute.readonly' + in node_data['serviceAccounts'][0]['scopes']) + def test_create_node_with_metadata(self): node_name = 'node-name' image = self.driver.ex_get_image('debian-7')
