commit: 4324b82edd9d69a27b1db9f4272564ddc51a75dd
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Feb 22 20:27:14 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Feb 22 20:27:14 2021 +0000
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=4324b82e
Add RunPkgCheck and CheckPkgCheckLogs
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 11 ++-
buildbot_gentoo_ci/db/model.py | 2 +-
buildbot_gentoo_ci/db/projects.py | 6 +-
buildbot_gentoo_ci/steps/builders.py | 120 +++++++++++++++++++++++++++--
4 files changed, 129 insertions(+), 10 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index 3c77762..cfe4131 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -92,13 +92,15 @@ def run_build_request():
# set needed Propertys
f.addStep(builders.SetupPropertys())
# Clean and add new /etc/portage
+ #FIXME: Is don't like symlinks
f.addStep(buildbot_steps.RemoveDirectory(dir="portage",
workdir='/etc/'))
f.addStep(buildbot_steps.MakeDirectory(dir="portage",
workdir='/etc/'))
# setup the profile
- f.addStep(buildbot_steps.MakeDirectory(dir="make.profile",
- workdir='/etc/portage/'))
+ #NOTE: pkgcheck do not support it as a dir
+ #f.addStep(buildbot_steps.MakeDirectory(dir="make.profile",
+ # workdir='/etc/portage/'))
f.addStep(builders.SetMakeProfile())
# setup repos.conf dir
f.addStep(buildbot_steps.MakeDirectory(dir="repos.conf",
@@ -132,4 +134,9 @@ def run_build_request():
# check log
# setup make.conf if build id has changes make.conf as dict from
SetMakeConf
# setup package.* env if build id has changes
+ # setup pkgcheck.conf if needed
+ #f.addStep(builders.SetPkgCheckConf())
+ # run pkgcheck if wanted
+ # check log
+ f.addStep(builders.RunPkgCheck())
return f
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 0a2c145..4d691ea 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -128,7 +128,7 @@ class Model(base.DBConnectorComponent):
sa.ForeignKey('repositorys.uuid', ondelete='CASCADE'),
nullable=False),
sa.Column('auto', sa.Boolean, default=False),
- sa.Column('pkgcheck', sa.Boolean, default=False),
+ sa.Column('pkgcheck', sa.Enum('package','full','none'),
default='none'),
)
# projects etc/portage settings
diff --git a/buildbot_gentoo_ci/db/projects.py
b/buildbot_gentoo_ci/db/projects.py
index eb8f727..9aeca50 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -160,12 +160,16 @@ class
ProjectsConnectorComponent(base.DBConnectorComponent):
)
def _row2dict_projects_repositorys(self, conn, row):
+ if row.pkgcheck == 'none':
+ pkgcheck = False
+ else:
+ pkgcheck=row.pkgcheck
return dict(
id=row.id,
project_uuid=row.project_uuid,
repository_uuid=row.repository_uuid,
auto=row.auto,
- pkgcheck=row.pkgcheck
+ pkgcheck=pkgcheck
)
def _row2dict_projects_portage(self, conn, row):
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index c0567c1..6d46ae6 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -4,6 +4,8 @@
import os
import re
+from portage.versions import catpkgsplit
+
from twisted.internet import defer
from twisted.python import log
@@ -75,6 +77,26 @@ def PersOutputOfEmerge(rc, stdout, stderr):
'emerge_output' : emerge_output
}
+def PersOutputOfPkgCheck(rc, stdout, stderr):
+ pkgcheck_output = {}
+ pkgcheck_output['rc'] = rc
+ #FIXME: Handling of stdout output
+ pkgcheck_xml_list = []
+ # split the lines
+ for line in stdout.split('\n'):
+ # pkgcheck output list
+ if line.startswith('<checks'):
+ pkgcheck_xml_list.append(line)
+ if line.startswith('<result'):
+ pkgcheck_xml_list.append(line)
+ if line.startswith('</checks'):
+ pkgcheck_xml_list.append(line)
+ pkgcheck_output['pkgcheck_xml'] = pkgcheck_xml_list
+ #FIXME: Handling of stderr output
+ return {
+ 'pkgcheck_output' : pkgcheck_output
+ }
+
class TriggerRunBuildRequest(BuildStep):
name = 'TriggerRunBuildRequest'
@@ -190,16 +212,31 @@ class SetMakeProfile(BuildStep):
project_data = self.getProperty('project_data')
profile_repository_data = yield
self.gentooci.db.repositorys.getRepositoryByUuid(project_data['profile_repository_uuid'])
makeprofiles_paths = []
+ #NOTE: pkgcheck don't support make.profile as a dir
+ # we only support one line in db
makeprofiles_data = yield
self.gentooci.db.projects.getAllProjectPortageByUuidAndDirectory(project_data['uuid'],
'make.profile')
for makeprofile in makeprofiles_data:
makeprofile_path = yield os.path.join(portage_repos_path,
profile_repository_data['name'], 'profiles', makeprofile['value'], '')
- makeprofiles_paths.append('../../..' + makeprofile_path)
- separator = '\n'
- makeprofile_path_string = separator.join(makeprofiles_paths)
+ # makeprofiles_paths.append('../../..' + makeprofile_path)
+ #separator = '\n'
+ #makeprofile_path_string = separator.join(makeprofiles_paths)
+ # yield self.build.addStepsAfterCurrentStep([
+ # steps.StringDownload(makeprofile_path_string + separator,
+ # workerdest="make.profile/parent",
+ # workdir='/etc/portage/')
+ # ])
+ #NOTE: pkgcheck profile link
+ shell_commad_list = [
+ 'ln',
+ '-s'
+ ]
+ shell_commad_list.append(makeprofile_path)
+ shell_commad_list.append('/etc/portage/make.profile')
yield self.build.addStepsAfterCurrentStep([
- steps.StringDownload(makeprofile_path_string + separator,
- workerdest="make.profile/parent",
- workdir='/etc/portage/')
+ steps.ShellCommandNewStyle(
+ command=shell_commad_list,
+ workdir='/'
+ )
])
return SUCCESS
@@ -515,3 +552,74 @@ class CheckEmergeLogs(BuildStep):
if not self.step is None:
yield self.build.addStepsAfterCurrentStep(aftersteps_list)
return SUCCESS
+
+class RunPkgCheck(BuildStep):
+
+ name = 'RunPkgCheck'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ projectrepository_data = self.getProperty('projectrepository_data')
+ if not projectrepository_data['pkgcheck']:
+ return SUCCESS
+ self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+ project_data = self.getProperty('project_data')
+ portage_repos_path = self.getProperty('portage_repos_path')
+ repository_data = yield
self.gentooci.db.repositorys.getRepositoryByUuid(projectrepository_data['repository_uuid'])
+ repository_path = yield os.path.join(portage_repos_path,
repository_data['name'])
+ cpv = self.getProperty("cpv")
+ c = yield catpkgsplit(cpv)[0]
+ p = yield catpkgsplit(cpv)[1]
+ shell_commad_list = [
+ 'pkgcheck',
+ 'scan',
+ '-v'
+ ]
+ shell_commad_list.append('-R')
+ shell_commad_list.append('XmlReporter')
+ aftersteps_list = []
+ if projectrepository_data['pkgcheck'] == 'full':
+ pkgcheck_workdir = yield os.path.join(repository_path, '')
+ else:
+ pkgcheck_workdir = yield os.path.join(repository_path, c, p, '')
+ aftersteps_list.append(
+ steps.SetPropertyFromCommandNewStyle(
+ command=shell_commad_list,
+ strip=True,
+ extract_fn=PersOutputOfPkgCheck,
+ workdir=pkgcheck_workdir
+ ))
+ aftersteps_list.append(CheckPkgCheckLogs())
+ yield self.build.addStepsAfterCurrentStep(aftersteps_list)
+ return SUCCESS
+
+class CheckPkgCheckLogs(BuildStep):
+
+ name = 'CheckPkgCheckLogs'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ #@defer.inlineCallbacks
+ def run(self):
+ self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+ project_data = self.getProperty('project_data')
+ pkgcheck_output = self.getProperty('pkgcheck_output')
+ print(pkgcheck_output)
+ #FIXME:
+ # Perse the logs
+ # tripp irc request with pkgcheck info
+ return SUCCESS
