commit: 5176761ad56557e45359729d7b2989dadeace7b5 Author: Magnus Granberg <zorry <AT> gentoo <DOT> org> AuthorDate: Thu Mar 11 01:14:02 2021 +0000 Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org> CommitDate: Thu Mar 11 01:14:02 2021 +0000 URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=5176761a
Add GitPuller to change_source and change cpv_changes to git_changes Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org> buildbot_gentoo_ci/config/change_source.py | 18 ++++++ buildbot_gentoo_ci/config/schedulers.py | 40 +++++++++---- buildbot_gentoo_ci/db/model.py | 10 +++- buildbot_gentoo_ci/db/repositorys.py | 48 ++++++++++----- buildbot_gentoo_ci/steps/update_db.py | 95 +++++++++++++++--------------- 5 files changed, 134 insertions(+), 77 deletions(-) diff --git a/buildbot_gentoo_ci/config/change_source.py b/buildbot_gentoo_ci/config/change_source.py new file mode 100644 index 0000000..454ed4a --- /dev/null +++ b/buildbot_gentoo_ci/config/change_source.py @@ -0,0 +1,18 @@ +# Copyright 2021 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +from buildbot.plugins import changes, util + +#FIXME: +# Get the repositorys info from the gentoo-ci db +def gentoo_change_source(cs=[]): + cs.append(changes.GitPoller( + repourl='https://github.com/gentoo/gentoo.git', + branches=True, + workdir= 'repositorys' + '/gentoo.git/', + pollInterval=240, + pollRandomDelayMin=20, + pollRandomDelayMax=60, + project='gentoo' + )) + return cs diff --git a/buildbot_gentoo_ci/config/schedulers.py b/buildbot_gentoo_ci/config/schedulers.py index 7cfb122..0cbec96 100644 --- a/buildbot_gentoo_ci/config/schedulers.py +++ b/buildbot_gentoo_ci/config/schedulers.py @@ -4,7 +4,7 @@ from buildbot.plugins import schedulers, util @util.renderer -def builderUpdateDbNames(self, props): +def builderUpdateDbNames(props): builders = set() for f in props.files: if f.endswith('.ebuild'): @@ -12,22 +12,38 @@ def builderUpdateDbNames(self, props): return list(builders) @util.renderer -def cpvUpdateDb(props): - cpv_changes = [] - for f in props.files: - if f.endswith('.ebuild'): - cppv = f.split('.eb', 0) - cpv = cppv.split('/', 0) + '/' + cppv.split('/', 2) - if not cpv in cpv_changes: - cpv_changes.append(cpv) - return cpv_changes +def gitUpdateDb(props): + git_changes = [] + print(props.changes) + for k in props.changes: + change_data = {} + print(k) + change_data['cpvs'] = [] + for v in k['files']: + if v.endswith('.ebuild'): + c = v.split('/')[0] + pv = v.split('/')[2][:-7] + cpv = c + '/' + pv + print(cpv) + change_data['cpvs'].append(cpv) + if k['repository'].endswith('.git'): + for v in k['repository'].split('/'): + if v.endswith('.git'): + change_data['repository'] = v[:-4] + change_data['author'] = k['author'] + change_data['committer'] = k['committer'] + change_data['comments'] = k['comments'] + change_data['revision'] = k['revision'] + git_changes.append(change_data) + print(git_changes) + return git_changes def gentoo_schedulers(): scheduler_update_db = schedulers.SingleBranchScheduler( name='scheduler_update_db', treeStableTimer=60, properties = { - 'cpv_changes' : cpvUpdateDb, + 'git_changes' : gitUpdateDb, }, builderNames = builderUpdateDbNames, change_filter=util.ChangeFilter(branch='master'), @@ -60,7 +76,7 @@ def gentoo_schedulers(): builderNames=["run_build_request"]) s = [] s.append(test_updatedb) - #s.append(scheduler_update_db) + s.append(scheduler_update_db) s.append(update_cpv_data) s.append(update_v_data) s.append(build_request_data) diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py index 2221ba4..939059c 100644 --- a/buildbot_gentoo_ci/db/model.py +++ b/buildbot_gentoo_ci/db/model.py @@ -71,14 +71,15 @@ class Model(base.DBConnectorComponent): # description of the repository sa.Column('description', sa.Text, nullable=True), sa.Column('mirror_url', sa.String(255), nullable=True), + sa.Column('type', sa.Enum('gitpuller'), nullable=False, default='gitpuller'), sa.Column('auto', sa.Boolean, default=False), sa.Column('enabled', sa.Boolean, default=False), sa.Column('ebuild', sa.Boolean, default=False), ) # Use by GitPoller - repository_gitpuller = sautils.Table( - "repository_gitpuller", metadata, + repositorys_gitpullers = sautils.Table( + "repositorys_gitpullers", metadata, # unique id per repository sa.Column('id', sa.Integer, primary_key=True), sa.Column('repository_uuid', sa.String(36), @@ -86,7 +87,10 @@ class Model(base.DBConnectorComponent): nullable=False), sa.Column('project', sa.String(255), nullable=False, default='gentoo'), sa.Column('url', sa.String(255), nullable=False), - sa.Column('branche', sa.String(255), nullable=False, default='master'), + sa.Column('branches', sa.String(255), nullable=False, default='all'), + sa.Column('poll_interval', sa.Integer, nullable=False, default=600), + sa.Column('poll_random_delay_min', sa.Integer, nullable=False, default=600), + sa.Column('poll_random_delay_max', sa.Integer, nullable=False, default=600), ) projects = sautils.Table( diff --git a/buildbot_gentoo_ci/db/repositorys.py b/buildbot_gentoo_ci/db/repositorys.py index 4660530..6a4ef83 100644 --- a/buildbot_gentoo_ci/db/repositorys.py +++ b/buildbot_gentoo_ci/db/repositorys.py @@ -25,16 +25,18 @@ from buildbot.db import base class RepositorysConnectorComponent(base.DBConnectorComponent): - def getRepositorys(self): + @defer.inlineCallbacks + def getAllRepositorysByEnableAuto(self): def thd(conn): tbl = self.db.model.repositorys q = tbl.select() q = q.where(tbl.c.enable == 1, tbl.c.auto == 1 ) - r = conn.execute(q) - return [dict(row) for row in r.fetchall()] - return self.db.pool.do(thd) + return [self._row2dict(conn, row) + for row in conn.execute(q).fetchall()] + res = yield self.db.pool.do(thd) + return res @defer.inlineCallbacks def getRepositoryByName(self, name): @@ -64,18 +66,19 @@ class RepositorysConnectorComponent(base.DBConnectorComponent): res = yield self.db.pool.do(thd) return res - def getRepositorysGitPoller(self): + @defer.inlineCallbacks + def getGitPollerByUuid(self, uuid): def thd(conn): - tblr = self.db.model.repositorys - tblrg = self.db.model.repository_gitpuller - from_clause = tblr.join(tblg, tblr.c.uuid == tblrg.c.repository_uuid) - q = sa.select([tblrg]).select_from( - from_clause).where(tblr.c.enabled == 1, - tblr.c.auto == 1 - ) - r = conn.execute(q) - return [dict(row) for row in r.fetchall()] - return self.db.pool.do(thd) + tbl = self.db.model.repositorys_gitpullers + q = tbl.select() + q = q.where(tbl.c.repository_uuid == uuid) + res = conn.execute(q) + row = res.fetchone() + if not row: + return None + return self._row2dict_gitpuller(conn, row) + res = yield self.db.pool.do(thd) + return res def _row2dict(self, conn, row): return dict( @@ -85,5 +88,18 @@ class RepositorysConnectorComponent(base.DBConnectorComponent): mirror_url=row.mirror_url, auto=row.auto, enabled=row.enabled, - ebuild=row.ebuild + ebuild=row.ebuild, + type=row.type + ) + + def _row2dict_gitpuller(self, conn, row): + return dict( + id=row.id, + repository_uuid=row.repository_uuid, + project=row.project, + url=row.url, + branches=row.branches, + poll_interval=row.poll_interval, + poll_random_delay_min=row.poll_random_delay_min, + poll_random_delay_max=row.poll_random_delay_max ) diff --git a/buildbot_gentoo_ci/steps/update_db.py b/buildbot_gentoo_ci/steps/update_db.py index 3f20cb9..94d0b4f 100644 --- a/buildbot_gentoo_ci/steps/update_db.py +++ b/buildbot_gentoo_ci/steps/update_db.py @@ -37,18 +37,22 @@ class GetDataGentooCiProject(BuildStep): if self.profile_repository_data is None: log.err('No data for repository in the database') return FAILURE - #self.repository = self.getProperty("repository") - self.repository = 'gentoo' - self.repository_data = yield self.gentooci.db.repositorys.getRepositoryByName(self.repository) print(self.project_data) print(self.project_repository_data) print(self.profile_repository_data) - print(self.getProperty("cpv_changes")) - print(self.repository_data) + print(self.getProperty("git_changes")) + print(self.getProperty("repository")) + repository = False + self.repository_data = False + if self.getProperty("repository").endswith('.git'): + for v in self.getProperty("repository").split('/'): + if v.endswith('.git'): + repository = v[:-4] + if repository: + self.repository_data = yield self.gentooci.db.repositorys.getRepositoryByName(repository) self.setProperty("project_data", self.project_data, 'project_data') self.setProperty("project_repository_data", self.project_repository_data, 'project_repository_data') self.setProperty("profile_repository_data", self.profile_repository_data, 'profile_repository_data') - self.setProperty("cpv_changes", self.getProperty("cpv_changes"), 'cpv_changes') self.setProperty("repository_data", self.repository_data, 'repository_data') return SUCCESS @@ -82,12 +86,9 @@ class CheckPathGentooCiProject(BuildStep): is_dir = False success = False print("isdir(%s): %s" %(x, is_dir)) + print(self.getProperty("builddir")) if not success: return FAILURE - self.setProperty("project_data", self.project_data, 'project_data') - self.setProperty("project_repository_data", self.project_repository_data, 'project_repository_data') - self.setProperty("cpv_changes", self.getProperty("cpv_changes"), 'cpv_changes') - self.setProperty("repository_data", self.repository_data, 'repository_data') return SUCCESS class CheckProjectGentooCiProject(BuildStep): @@ -111,9 +112,6 @@ class CheckProjectGentooCiProject(BuildStep): print("project portage conf has error %s" %(str(e))) return FAILURE self.setProperty("config_root", self.config_root, 'config_root') - self.setProperty("project_data", self.project_data, 'project_data') - self.setProperty("cpv_changes", self.getProperty("cpv_changes"), 'cpv_changes') - self.setProperty("repository_data", self.getProperty("repository_data"), 'repository') return SUCCESS class CheckCPVGentooCiProject(BuildStep): @@ -122,41 +120,46 @@ class CheckCPVGentooCiProject(BuildStep): @defer.inlineCallbacks def run(self): - #self.cpv_changes = self.getProperty("cpv_changes") - self.cpv_changes = [] - self.cpv_changes.append('dev-python/django-3.1.7') - self.cpv_changes.append('dev-python/scrypt-0.8.16') - print(self.cpv_changes) - print(self.getProperty("repository_data")) - # check if cpv_change is a string or a list - if isinstance(self.cpv_changes, list): - self.cpv_list = self.cpv_changes - else: - self.cpv_list = [] - self.cpv_list.append(self.cpv_changes) + self.git_changes = self.getProperty("git_changes") + print(self.git_changes) + # check if git_change is a string or a list + if not isinstance(self.git_changes, list): + return FAILURE self.success = True addStepUpdateCPVData = [] - for cpv in sorted(self.cpv_list): - # check that cpv is valied - if catpkgsplit(cpv) is None: - log.msg("%s is not vaild package name" % cpv) - self.success = False - else: - # call update_cpv_data - addStepUpdateCPVData.append( - steps.Trigger( - schedulerNames=['update_cpv_data'], - waitForFinish=False, - updateSourceStamp=False, - set_properties={ - 'cpv' : cpv, - 'config_root' : self.getProperty("config_root"), - 'project_data' : self.getProperty("project_data"), - 'repository_data' : self.getProperty("repository_data"), - } - ) - ) - yield self.build.addStepsAfterCurrentStep(addStepUpdateCPVData) + for change_data in self.git_changes: + # make a trigger for all cpv in the list + for cpv in change_data['cpvs']: + # check that cpv is valied + if catpkgsplit(cpv) is None: + log.msg("%s is not vaild package name" % cpv) + self.success = False + else: + if change_data['repository'] != self.getProperty("repository_data")['name']: + log.msg("%s don't match" % change_data['repository']) + self.success = False + else: + revision_data = {} + revision_data['author'] = change_data['author'] + revision_data['committer'] = change_data['committer'] + revision_data['comments'] = change_data['comments'] + revision_data['revision'] = change_data['revision'] + # call update_cpv_data + addStepUpdateCPVData.append( + steps.Trigger( + schedulerNames=['update_cpv_data'], + waitForFinish=False, + updateSourceStamp=False, + set_properties={ + 'cpv' : cpv, + 'config_root' : self.getProperty("config_root"), + 'project_data' : self.getProperty("project_data"), + 'repository_data' : self.getProperty("repository_data"), + 'revision_data' : revision_data, + } + ) + ) + yield self.build.addStepsAfterCurrentStep(addStepUpdateCPVData) if self.success is False: return FAILURE return SUCCESS
