commit: 8bdcac12a5c3a41d8321a55b8fef36355dc0f6e7
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Fri Apr 2 22:11:49 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Fri Apr 2 22:11:49 2021 +0000
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=8bdcac12
Add GetAuxMetadata and use that to get aux metadata
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/portage.py | 63 ++++++++++++++++++++++++++++++++---
buildbot_gentoo_ci/steps/update_db.py | 17 +++++++++-
buildbot_gentoo_ci/steps/version.py | 12 ++++---
3 files changed, 83 insertions(+), 9 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/portage.py
b/buildbot_gentoo_ci/steps/portage.py
index 293b862..d66c952 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -10,6 +10,7 @@ from portage import _encodings
from portage import _unicode_encode
from portage import _parse_eapi_ebuild_head, eapi_is_supported
from portage.versions import cpv_getversion, pkgsplit, catpkgsplit
+from portage import portdbapi
from twisted.internet import defer
from twisted.python import log
@@ -249,7 +250,7 @@ class SetMakeConf(BuildStep):
makeconf_list.append('DISTDIR="/var/cache/portage/distfiles"')
makeconf_list.append('PORTAGE_ELOG_CLASSES="*"')
# We need echo:info to get the logfile name
- makeconf_list.append('PORTAGE_ELOG_SYSTEM="save echo:info"')
+ makeconf_list.append('PORTAGE_ELOG_SYSTEM="save:* echo:info"')
# add ACCEPT_KEYWORDS from the project_data info
keyword_data = yield
self.gentooci.db.keywords.getKeywordById(project_data['keyword_id'])
if project_data['status'] == 'unstable':
@@ -395,7 +396,8 @@ class SetReposConfLocal(BuildStep):
repos_conf_path = yield os.path.join('etc', 'portage', 'repos.conf')
repos_conf_default_path = yield os.path.join(repos_conf_path,
'default.conf')
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
- self.repository_basedir =
self.gentooci.config.project['repository_basedir']
+ # the path should be set in the confg
+ self.repository_basedir2 = '/home/repos2/'
if not os.path.isfile(repos_conf_default_path):
# setup the default.conf
repos_conf_data = yield
self.gentooci.db.projects.getProjectPortageByUuidAndDirectory(self.getProperty('project_data')['uuid'],
'repos.conf')
@@ -409,7 +411,7 @@ class SetReposConfLocal(BuildStep):
yield WriteTextToFile(repos_conf_default_path, default_conf)
repos_conf_repository_path = yield os.path.join(repos_conf_path,
self.getProperty("repository_data")['name'] + '.conf')
if not os.path.isfile(repos_conf_repository_path):
- repository_path = yield os.path.join(self.getProperty("builddir"),
self.repository_basedir, self.getProperty("repository_data")['name'])
+ repository_path = yield os.path.join(self.repository_basedir2,
self.getProperty("repository_data")['name'])
repository_conf = []
repository_conf.append('[' +
self.getProperty("repository_data")['name'] + ']')
repository_conf.append('location = ' + repository_path)
@@ -444,7 +446,7 @@ class SetMakeConfLocal(BuildStep):
makeconf_list.append('ACCEPT_KEYWORDS="~amd64 amd64"')
makeconf_list.append('EMERGE_DEFAULT_OPTS=""')
makeconf_list.append('ABI_X86="32 64"')
- makeconf_list.append('FEATURES=""')
+ makeconf_list.append('FEATURES="sandbox"')
yield WriteTextToFile(make_conf_path, makeconf_list)
return SUCCESS
@@ -530,3 +532,56 @@ class SetEnvForEbuildSH(BuildStep):
))
yield self.build.addStepsAfterCurrentStep(addStepEbuildSH)
return SUCCESS
+
+class GetAuxMetadata(BuildStep):
+
+ name = 'GetAuxMetadata'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ #@defer.inlineCallbacks
+ def thd_getAuxDbKeys(self):
+ auxdbs = self.myportdb.aux_get(self.getProperty("cpv"), auxdbkeys,
myrepo=self.getProperty("repository_data")['name'])
+ return auxdbs
+
+ @defer.inlineCallbacks
+ def run(self):
+ config_root = yield os.path.join(self.getProperty("builddir"), '')
+ # setup mysettings and myportdb
+ mysettings = yield portage_config(config_root = config_root)
+ self.myportdb = yield portdbapi(mysettings=mysettings)
+ auxdbs = yield self.thd_getAuxDbKeys()
+ metadata = None
+ NoSplit = []
+ NoSplit.append('DESCRIPTION')
+ # should have 22 lines
+ if len(auxdbkeys) != len(auxdbs) or not isinstance(auxdbs, list):
+ # number of lines is incorrect or not a list.
+ print("Lines don't match or not a list")
+ yield self.myportdb.close_caches()
+ yield portdbapi.portdbapi_instances.remove(self.myportdb)
+ #self.setProperty('auxdb', metadata, 'auxdb')
+ return FAILURE
+ # split all keys to list instead of speces
+ metadata = {}
+ i = 0
+ for key in auxdbkeys:
+ if auxdbs[i] == '':
+ metadata[key] = None
+ else:
+ if ' ' in auxdbs[i] and key not in NoSplit:
+ metadata[key] = auxdbs[i].split(' ')
+ else:
+ metadata[key] = []
+ metadata[key].append(auxdbs[i])
+ i = i + 1
+ self.setProperty('auxdb', metadata, 'auxdb')
+ yield self.myportdb.close_caches()
+ yield portdbapi.portdbapi_instances.remove(self.myportdb)
+ return SUCCESS
diff --git a/buildbot_gentoo_ci/steps/update_db.py
b/buildbot_gentoo_ci/steps/update_db.py
index b84e256..3594c1b 100644
--- a/buildbot_gentoo_ci/steps/update_db.py
+++ b/buildbot_gentoo_ci/steps/update_db.py
@@ -140,11 +140,20 @@ class UpdateRepos(BuildStep):
else:
raise AssertionError('Unknown merge analysis result')
+ @defer.inlineCallbacks
+ def setchmod(self, path):
+ for root, dirs, files in os.walk(path):
+ for d in dirs:
+ yield os.chmod(os.path.join(root, d), 0o0755)
+ for f in files:
+ yield os.chmod(os.path.join(root, f), 0o0644)
+
@defer.inlineCallbacks
def run(self):
#FIXME check HEAD agenst local and worker local tree so we don't
gitpull evrytime
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
- self.repository_basedir =
self.gentooci.config.project['repository_basedir']
+ #self.repository_basedir =
self.gentooci.config.project['repository_basedir']
+ self.repository_basedir = yield os.path.join('/home', 'repos2')
self.profile_repository_path = yield
os.path.join(self.repository_basedir,
self.getProperty("profile_repository_data")['name'])
repo_path = yield
pygit2.discover_repository(self.profile_repository_path)
print(repo_path)
@@ -153,6 +162,9 @@ class UpdateRepos(BuildStep):
else:
repo = yield pygit2.Repository(repo_path)
yield self.gitPull(repo)
+ # chmod and chown
+ yield self.setchmod(self.profile_repository_path)
+ #yield os.chown(self.profile_repository_path, 'buildbot', 'portage')
if self.getProperty("profile_repository_data")['name'] !=
self.getProperty("repository_data")['name']:
self.repository_path = yield os.path.join(self.repository_basedir,
self.getProperty("repository_data")['name'])
repo_path = yield pygit2.discover_repository(self.repository_path)
@@ -161,6 +173,9 @@ class UpdateRepos(BuildStep):
else:
repo = yield pygit2.Repository(repo_path)
yield self.gitPull(repo)
+ # chmod and chown
+ yield self.setchmod(self.profile_repository_path)
+ #yield os.chown(self.repository_path, 'buildbot', 'portage')
return SUCCESS
class TriggerCheckForCPV(BuildStep):
diff --git a/buildbot_gentoo_ci/steps/version.py
b/buildbot_gentoo_ci/steps/version.py
index 4c35914..426ddd4 100644
--- a/buildbot_gentoo_ci/steps/version.py
+++ b/buildbot_gentoo_ci/steps/version.py
@@ -131,7 +131,6 @@ class AddVersionKeyword(BuildStep):
return SUCCESS
print(auxdb)
for keyword in auxdb:
- print(keyword)
status = 'stable'
if keyword[0] in ["~"]:
keyword = keyword[1:]
@@ -170,7 +169,8 @@ class CheckPathHash(BuildStep):
def run(self):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
self.repository_basedir =
self.gentooci.config.project['repository_basedir']
- self.repository_path = yield os.path.join(self.repository_basedir,
self.getProperty("repository_data")['name'])
+ self.repository_path = yield os.path.join('/home', 'repos2',
self.getProperty("repository_data")['name'])
+ #self.repository_path = yield os.path.join(self.repository_basedir,
self.getProperty("repository_data")['name'])
self.cp_path = yield pkgsplit(self.getProperty("cpv"))[0]
self.file_name = yield self.getProperty("package_data")['name'] + '-'
+ self.getProperty("version") + '.ebuild'
self.ebuild_file = yield os.path.join(self.repository_path,
self.cp_path, self.file_name)
@@ -256,7 +256,9 @@ class CheckV(BuildStep):
if self.getProperty("ebuild_file") is not None and
self.getProperty("old_version_data") is not None:
if self.getProperty("ebuild_file_hash") !=
self.getProperty("old_version_data")['file_hash']:
addStepVData.append(GetCommitdata())
- addStepVData.append(portage_steps.SetEnvForEbuildSH())
+ #FIXME: use GetAuxMetadata insted of bugy SetEnvForEbuildSH
+ #addStepVData.append(portage_steps.SetEnvForEbuildSH())
+ addStepVData.append(portage_steps.GetAuxMetadata())
addStepVData.append(AddVersion())
addStepVData.append(AddVersionKeyword())
addStepVData.append(TriggerBuildCheck())
@@ -265,7 +267,9 @@ class CheckV(BuildStep):
return SUCCESS
if self.getProperty("ebuild_file") is not None and
self.getProperty("old_version_data") is None:
addStepVData.append(GetCommitdata())
- addStepVData.append(portage_steps.SetEnvForEbuildSH())
+ #FIXME: use GetAuxMetadata insted of bugy SetEnvForEbuildSH
+ #addStepVData.append(portage_steps.SetEnvForEbuildSH())
+ addStepVData.append(portage_steps.GetAuxMetadata())
addStepVData.append(AddVersion())
addStepVData.append(AddVersionKeyword())
addStepVData.append(TriggerBuildCheck())
