commit: 17241c67856e8f7257cf4d8e5785f25c856a1118 Author: Magnus Granberg <zorry <AT> gentoo <DOT> org> AuthorDate: Mon Apr 13 23:27:48 2020 +0000 Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org> CommitDate: Mon Apr 13 23:27:48 2020 +0000 URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=17241c67
Add service builder with task update_git for it Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org> gosbs/builder/__init__.py | 0 gosbs/builder/manager.py | 104 +++++++++++++++++++++++++++ gosbs/builder/rpcapi.py | 123 ++++++++++++++++++++++++++++++++ gosbs/cmd/builder.py | 45 ++++++++++++ gosbs/common/portage_settings.py | 8 ++- gosbs/conf/__init__.py | 86 ++++++++++++++++++++-- gosbs/conf/{scheduler.py => builder.py} | 12 ++-- gosbs/conf/upgrade_levels.py | 2 +- gosbs/db/sqlalchemy/models.py | 6 +- gosbs/scheduler/project.py | 3 +- gosbs/tasks/builder/__init__.py | 24 +++++++ gosbs/tasks/builder/sub/__init__.py | 0 gosbs/tasks/builder/update_git.py | 67 +++++++++++++++++ requirements.txt | 14 +--- 14 files changed, 463 insertions(+), 31 deletions(-) diff --git a/gosbs/builder/__init__.py b/gosbs/builder/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gosbs/builder/manager.py b/gosbs/builder/manager.py new file mode 100644 index 0000000..950122b --- /dev/null +++ b/gosbs/builder/manager.py @@ -0,0 +1,104 @@ +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 Justin Santa Barbara +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Handles all processes relating to instances (guest vms). + +The :py:class:`ComputeManager` class is a :py:class:`nova.manager.Manager` that +handles RPC calls relating to creating instances. It is responsible for +building a disk image, launching it via the underlying virtualization driver, +responding to calls to check its state, attaching persistent storage, and +terminating it. + +""" +#import functools +#import json +from importlib import import_module +import sys + +from oslo_log import log as logging +import oslo_messaging as messaging +from oslo_service import periodic_task +from oslo_utils import timeutils +from openstack import connection + +from gosbs.builder import rpcapi as builder_rpcapi +from gosbs import rpc +#from gosbs import exception_wrapper +from gosbs import manager +from gosbs import objects +from gosbs.objects import base as obj_base +from gosbs.objects import fields +from gosbs.tasks import builder as builder_tasks +from gosbs.common.task import run_task + +import gosbs.conf + +CONF = gosbs.conf.CONF + +LOG = logging.getLogger(__name__) + +#get_notifier = functools.partial(rpc.get_notifier, service='scheduler') +#wrap_exception = functools.partial(exception_wrapper.wrap_exception, +# get_notifier=get_notifier, +# binary='gobs-scheduler') + +class BuilderManager(manager.Manager): + """Manages the running instances from creation to destruction.""" + + #target = messaging.Target(version='1.0') + + def __init__(self, *args, **kwargs): + """Load configuration options and connect to the hypervisor.""" + self.builder_rpcapi = builder_rpcapi.BuilderAPI() + + super(BuilderManager, self).__init__(service_name="Builder", + *args, **kwargs) + + + def init_host(self): + context = gosbs.context.get_admin_context() + + def pre_start_hook(self): + context = gosbs.context.get_admin_context() + self.openstack_conn = connection.Connection( + region_name = CONF.keystone.region_name, + auth=dict( + auth_url = CONF.keystone.auth_url, + username = CONF.keystone.username, + password = CONF.keystone.password, + project_id = CONF.keystone.project_id, + user_domain_id = CONF.keystone.user_domain_name), + scheduler_api_version = CONF.keystone.auth_version, + identity_interface= CONF.keystone.identity_interface) + self.service_ref = objects.Service.get_by_host_and_topic( + context, self.host, "builder") + builder_tasks.activete_all_tasks(context, self.service_ref.uuid) + + def reset(self): + LOG.info('Reloading builder RPC API') + builder_rpcapi.LAST_VERSION = None + self.builder_rpcapi = builder_rpcapi.BuilderAPI() + + @periodic_task.periodic_task + def update_git_task(self, context): + task_name = 'update_git' + LOG.debug("Runing task %s", task_name) + filters = { 'status' : 'waiting', + 'name' : task_name, + 'service_uuid' : self.service_ref.uuid, + } + run_task(context, filters, self.service_ref) diff --git a/gosbs/builder/rpcapi.py b/gosbs/builder/rpcapi.py new file mode 100644 index 0000000..1c1160f --- /dev/null +++ b/gosbs/builder/rpcapi.py @@ -0,0 +1,123 @@ +# Copyright 2013 Red Hat, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Client side of the scheduler RPC API. +""" + +from oslo_log import log as logging +import oslo_messaging as messaging +from oslo_serialization import jsonutils + +import gosbs.conf +from gosbs import context +from gosbs import exception +from gosbs.i18n import _ +from gosbs import objects +from gosbs.objects import base as objects_base +from gosbs.objects import service as service_obj +from gosbs import profiler +from gosbs import rpc + +CONF = gosbs.conf.CONF +RPC_TOPIC = "builder" + +LOG = logging.getLogger(__name__) +LAST_VERSION = None + +@profiler.trace_cls("rpc") +class BuilderAPI(object): + '''Client side of the compute rpc API. + + * 5.0 - Remove 4.x compatibility + ''' + + VERSION_ALIASES = { + 'rocky': '1.0', + } + + def __init__(self): + super(BuilderAPI, self).__init__() + target = messaging.Target(topic=RPC_TOPIC, version='1.0') + upgrade_level = CONF.upgrade_levels.builder + if upgrade_level == 'auto': + version_cap = self._determine_version_cap(target) + else: + version_cap = self.VERSION_ALIASES.get(upgrade_level, + upgrade_level) + serializer = objects_base.NovaObjectSerializer() + + # NOTE(danms): We need to poke this path to register CONF options + # that we use in self.get_client() + rpc.get_client(target, version_cap, serializer) + + default_client = self.get_client(target, version_cap, serializer) + self.router = rpc.ClientRouter(default_client) + + def _determine_version_cap(self, target): + global LAST_VERSION + if LAST_VERSION: + return LAST_VERSION + service_version = objects.Service.get_minimum_version( + context.get_admin_context(), 'gosbs-builder') + + history = service_obj.SERVICE_VERSION_HISTORY + + # NOTE(johngarbutt) when there are no nova-compute services running we + # get service_version == 0. In that case we do not want to cache + # this result, because we will get a better answer next time. + # As a sane default, return the current version. + if service_version == 0: + LOG.debug("Not caching compute RPC version_cap, because min " + "service_version is 0. Please ensure a nova-compute " + "service has been started. Defaulting to current " + "version.") + return history[service_obj.SERVICE_VERSION]['builder_rpc'] + + try: + version_cap = history[service_version]['builder_rpc'] + except IndexError: + LOG.error('Failed to extract compute RPC version from ' + 'service history because I am too ' + 'old (minimum version is now %(version)i)', + {'version': service_version}) + raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, + minver=service_version) + except KeyError: + LOG.error('Failed to extract compute RPC version from ' + 'service history for version %(version)i', + {'version': service_version}) + return target.version + LAST_VERSION = version_cap + LOG.info('Automatically selected compute RPC version %(rpc)s ' + 'from minimum service version %(service)i', + {'rpc': version_cap, + 'service': service_version}) + return version_cap + + # Cells overrides this + def get_client(self, target, version_cap, serializer): + if CONF.rpc_response_timeout > rpc.HEARTBEAT_THRESHOLD: + # NOTE(danms): If the operator has overridden RPC timeout + # to be longer than rpc.HEARTBEAT_THRESHOLD then configure + # the call monitor timeout to be the threshold to keep the + # failure timing characteristics that our code likely + # expects (from history) while allowing healthy calls + # to run longer. + cmt = rpc.HEARTBEAT_THRESHOLD + else: + cmt = None + return rpc.get_client(target, + version_cap=version_cap, + serializer=serializer, + call_monitor_timeout=cmt) diff --git a/gosbs/cmd/builder.py b/gosbs/cmd/builder.py new file mode 100644 index 0000000..071241e --- /dev/null +++ b/gosbs/cmd/builder.py @@ -0,0 +1,45 @@ +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Origin https://github.com/openstack/nova/blob/master/nova/cmd/compute.py +# Removed code that was not needed + +"""Starter script for Gosbs Builder.""" + +import shlex +import sys + +from oslo_log import log as logging + +from gosbs.builder import rpcapi as builder_rpcapi +import gosbs.conf +from gosbs import config +from gosbs import objects +from gosbs.objects import base as objects_base +from gosbs import service + +CONF = gosbs.conf.CONF + +def main(): + config.parse_args(sys.argv) + logging.setup(CONF, 'gosbs') + objects.register_all() + + objects.Service.enable_min_version_cache() + server = service.Service.create(binary='gosbs-builder', + topic=builder_rpcapi.RPC_TOPIC) + service.serve(server) + service.wait() diff --git a/gosbs/common/portage_settings.py b/gosbs/common/portage_settings.py index e5a58f5..f3a875d 100644 --- a/gosbs/common/portage_settings.py +++ b/gosbs/common/portage_settings.py @@ -17,12 +17,18 @@ from pathlib import Path import portage from oslo_log import log as logging - from gosbs import objects import gosbs.conf CONF = gosbs.conf.CONF + LOG = logging.getLogger(__name__) +def check_portage(context, project_repopath): + if not Path('/etc/portage').is_symlink(): + Path('/etc/portage').symlink_to(project_repopath + 'etc/portage') + else: + if Path('/etc/portage').resolve() != project_repopath + 'etc/portage': + pass def check_profile(context, project_repopath, project_metadata_db): profile_repo_db = objects.repo.Repo.get_by_uuid(context, project_metadata_db.project_profile_repo_uuid) diff --git a/gosbs/conf/__init__.py b/gosbs/conf/__init__.py index 05b2f36..d79e8a5 100644 --- a/gosbs/conf/__init__.py +++ b/gosbs/conf/__init__.py @@ -17,31 +17,107 @@ # have a central place where the config options of Nova can be maintained. # For more background see the blueprint "centralize-config-options" -# Origin https://github.com/openstack/nova/blob/master/nova/conf/__init__.py -# Import only what we need on gosbs - from oslo_config import cfg +#from nova.conf import api +#from nova.conf import availability_zone from gosbs.conf import base +from gosbs.conf import builder +#from nova.conf import cache +#from nova.conf import cinder +#from nova.conf import compute +#from nova.conf import conductor +#from nova.conf import configdrive +#from nova.conf import console +#from nova.conf import consoleauth from gosbs.conf import database +#from nova.conf import devices +#from nova.conf import ephemeral_storage +#from nova.conf import glance +#from nova.conf import guestfs +#from nova.conf import hyperv +#from nova.conf import imagecache +#from nova.conf import ironic +#from nova.conf import key_manager from gosbs.conf import keystone +#from nova.conf import libvirt +#from nova.conf import mks from gosbs.conf import netconf +#from nova.conf import neutron from gosbs.conf import notifications +#from nova.conf import novnc from gosbs.conf import paths +#from nova.conf import pci +#from nova.conf import placement +#from nova.conf import powervm +#from nova.conf import quota +#from nova.conf import rdp +#from nova.conf import remote_debug from gosbs.conf import rpc -from gosbs.conf import scheduler +#from gosbs.conf import scheduler +#from nova.conf import serial_console from gosbs.conf import service +#from gosbs.conf import service_token +#from gosbs.conf import servicegroup +#from nova.conf import spice from gosbs.conf import upgrade_levels +#from nova.conf import vendordata +#from nova.conf import vmware +#from nova.conf import vnc +#from nova.conf import workarounds +#from nova.conf import wsgi +#from nova.conf import xenserver +#from nova.conf import zvm CONF = cfg.CONF +#api.register_opts(CONF) +#availability_zone.register_opts(CONF) base.register_opts(CONF) +builder.register_opts(CONF) +#cache.register_opts(CONF) +#cinder.register_opts(CONF) +#compute.register_opts(CONF) +#conductor.register_opts(CONF) +#configdrive.register_opts(CONF) +#console.register_opts(CONF) +#consoleauth.register_opts(CONF) database.register_opts(CONF) +#devices.register_opts(CONF) +#ephemeral_storage.register_opts(CONF) +#glance.register_opts(CONF) +#guestfs.register_opts(CONF) +#hyperv.register_opts(CONF) +#mks.register_opts(CONF) +#imagecache.register_opts(CONF) +#ironic.register_opts(CONF) +#key_manager.register_opts(CONF) keystone.register_opts(CONF) +#libvirt.register_opts(CONF) netconf.register_opts(CONF) +#neutron.register_opts(CONF) notifications.register_opts(CONF) +#novnc.register_opts(CONF) paths.register_opts(CONF) +#pci.register_opts(CONF) +#placement.register_opts(CONF) +#powervm.register_opts(CONF) +#quota.register_opts(CONF) +#rdp.register_opts(CONF) rpc.register_opts(CONF) -scheduler.register_opts(CONF) +#scheduler.register_opts(CONF) +#serial_console.register_opts(CONF) service.register_opts(CONF) +#service_token.register_opts(CONF) +#servicegroup.register_opts(CONF) +#spice.register_opts(CONF) upgrade_levels.register_opts(CONF) +#vendordata.register_opts(CONF) +#vmware.register_opts(CONF) +#vnc.register_opts(CONF) +#workarounds.register_opts(CONF) +#wsgi.register_opts(CONF) +#xenserver.register_opts(CONF) +#zvm.register_opts(CONF) + +#remote_debug.register_cli_opts(CONF) diff --git a/gosbs/conf/scheduler.py b/gosbs/conf/builder.py similarity index 81% rename from gosbs/conf/scheduler.py rename to gosbs/conf/builder.py index 7b38d53..c3cb9b5 100644 --- a/gosbs/conf/scheduler.py +++ b/gosbs/conf/builder.py @@ -1,4 +1,6 @@ -# Copyright 1999-2020 Gentoo Authors +# Copyright 2015 OpenStack Foundation +# All Rights Reserved. +# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -13,8 +15,8 @@ from oslo_config import cfg -scheduler_group = cfg.OptGroup(name="scheduler", - title="Scheduler configuration") +scheduler_group = cfg.OptGroup(name="builder", + title="Builder configuration") scheduler_opts = [ cfg.StrOpt( @@ -23,9 +25,9 @@ scheduler_opts = [ Explicitly specify the mirror git url. """), cfg.StrOpt( - 'db_project_repo', + 'project', help=""" -Explicitly specify the database project repo. +Explicitly specify the project. """), ] diff --git a/gosbs/conf/upgrade_levels.py b/gosbs/conf/upgrade_levels.py index 767ff59..5024d61 100644 --- a/gosbs/conf/upgrade_levels.py +++ b/gosbs/conf/upgrade_levels.py @@ -33,7 +33,7 @@ help with a possible future backport issue. # TODO(sneti): Add default=auto for compute upgrade_levels_opts = [ - cfg.StrOpt('compute', + cfg.StrOpt('builder', help=""" Compute RPC API version cap. diff --git a/gosbs/db/sqlalchemy/models.py b/gosbs/db/sqlalchemy/models.py index be3e48f..119369c 100644 --- a/gosbs/db/sqlalchemy/models.py +++ b/gosbs/db/sqlalchemy/models.py @@ -15,12 +15,8 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - -# Origin https://github.com/openstack/nova/blob/master/nova/db/sqlalchemy/models.py -# Only small part is left from the origin - """ -SQLAlchemy models for gosbs data. +SQLAlchemy models for nova data. """ import uuid diff --git a/gosbs/scheduler/project.py b/gosbs/scheduler/project.py index 5c49be6..401f802 100644 --- a/gosbs/scheduler/project.py +++ b/gosbs/scheduler/project.py @@ -20,8 +20,9 @@ CONF = gosbs.conf.CONF LOG = logging.getLogger(__name__) def get_project(context, service_repo_db): - project_db = objects.project.Project.get_by_name(context, CONF.scheduler.db_project_repo) + project_db = objects.project.Project.get_by_name(context, CONF.builder.project) project_metadata_db = objects.project_metadata.ProjectMetadata.get_by_uuid(context, project_db.uuid) + print(project_db) filters = { 'project_uuid' : project_db.uuid, 'repo_uuid' : service_repo_db.repo_uuid, } diff --git a/gosbs/tasks/builder/__init__.py b/gosbs/tasks/builder/__init__.py new file mode 100644 index 0000000..bb93406 --- /dev/null +++ b/gosbs/tasks/builder/__init__.py @@ -0,0 +1,24 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from datetime import datetime + +from oslo_log import log as logging + +from gosbs import objects +from gosbs.common.task import check_task_db + +LOG = logging.getLogger(__name__) + +def activete_all_tasks(context, service_uuid): + # Tasks + check_task_db(context, 'update_git', datetime(1, 1, 1, 0, 5, 0, 0), True, service_uuid) diff --git a/gosbs/tasks/builder/sub/__init__.py b/gosbs/tasks/builder/sub/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gosbs/tasks/builder/update_git.py b/gosbs/tasks/builder/update_git.py new file mode 100644 index 0000000..cd464fa --- /dev/null +++ b/gosbs/tasks/builder/update_git.py @@ -0,0 +1,67 @@ +# Copyright 1999-2020 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +import sys + +from oslo_log import log as logging +from gosbs import objects +from gosbs.common.git import check_git_repo +from gosbs.common.portage_settings import check_profile, check_portage +import gosbs.conf + +CONF = gosbs.conf.CONF +LOG = logging.getLogger(__name__) + +def update_repo_git_thread(context, service_uuid, repo_db): + repo_dict = { 'repo_uuid' : repo_db.uuid, + 'repo_name' : repo_db.name, + 'repo_url' : CONF.builder.git_mirror_url + '/' + repo_db.name + '.git', + 'repo_type' : repo_db.repo_type, + 'repo_path' : CONF.repopath + '/' + repo_db.name + '.git', + 'history' : False, + } + filters = { + 'repo_uuid' : repo_db.uuid, + 'service_uuid' : service_uuid, + } + service_repo_db = objects.service_repo.ServiceRepo.get_by_filters(context, filters=filters) + if service_repo_db is None: + service_repo_db = objects.service_repo.ServiceRepo() + service_repo_db.repo_uuid = repo_db.uuid + service_repo_db.service_uuid = service_uuid + service_repo_db.auto = repo_db.auto + service_repo_db.status = 'waiting' + service_repo_db.create(context) + if service_repo_db.status == 'waiting': + service_repo_db.status = 'in-progress' + service_repo_db.save(context) + succes = check_git_repo(repo_dict) + if succes: + service_repo_db.status = 'completed' + else: + service_repo_db.status = 'failed' + service_repo_db.save(context) + return succes + return True + +def task(context, service_uuid): + project_db = objects.project.Project.get_by_name(context, CONF.builder.project) + project_metadata_db = objects.project_metadata.ProjectMetadata.get_by_uuid(context, project_db.uuid) + repo_db = objects.repo.Repo.get_by_uuid(context, project_metadata_db.project_repo_uuid) + succes = update_repo_git_thread(context, service_uuid, repo_db) + filters = { + 'project_uuid' : project_db.uuid, + } + if succes: + project_repopath = CONF.repopath + '/' + repo_db.name + '.git/' + CONF.builder.project + '/' + check_portage(context, project_repopath) + check_profile(context, project_repopath, project_metadata_db) + for project_repo_db in objects.project_repo.ProjectRepoList.get_all(context, filters): + repo_db = objects.repo.Repo.get_by_uuid(context, project_repo_db.repo_uuid) + succes = update_repo_git_thread(context, service_uuid, repo_db) + return + #with futurist.GreenThreadPoolExecutor(max_workers=1) as executor: + # Start the load operations and mark each future with its URL + # for cp in cp_list: + # future = executor.submit(update_cpv_db_thread, context, cp, repo_uuid, project_uuid) + # print(future.result()) diff --git a/requirements.txt b/requirements.txt index 34d697a..486d56e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1,6 @@ -pbr!=2.1.0,>=2.0.0 # Apache-2.0 -SQLAlchemy>=1.2.19 # MIT -keystonemiddleware>=4.20.0 # Apache-2.0 -greenlet>=0.4.10,!=0.4.14 # MIT -keystoneauth1>=3.16.0 # Apache-2.0 -oslo.config>=6.1.0 # Apache-2.0 -oslo.context>=2.21.0 # Apache-2.0 -oslo.log>=3.36.0 # Apache-2.0 -oslo.serialization!=2.19.1,>=2.21.1 # Apache-2.0 -oslo.utils>=3.40.2 # Apache-2.0 +#keystonemiddleware>=4.20.0 # Apache-2.0 oslo.db>=4.44.0 # Apache-2.0 -oslo.messaging>=10.3.0 # Apache-2.0 oslo.policy>=2.3.0 # Apache-2.0 -oslo.i18n>=3.15.3 # Apache-2.0 oslo.service>=1.40.1 # Apache-2.0 -oslo.middleware>=3.31.0 # Apache-2.0 oslo.versionedobjects>=1.35.0 # Apache-2.0 openstacksdk>=0.35.0 # Apache-2.0