commit: f2b047e20193e5cafe00e71846ba84df5fa3cc59 Author: Magnus Granberg <zorry <AT> gentoo <DOT> org> AuthorDate: Sat May 9 14:58:13 2015 +0000 Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org> CommitDate: Sat May 9 14:58:13 2015 +0000 URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=f2b047e2
Add repoman and qa info to db, fix buildjob check and some smaller fix. sql/structure_dump.sql | 93 +++++++++++++++++++++++++------------------------- tbc/pym/build_log.py | 24 ++++++------- tbc/pym/check_setup.py | 15 +++++--- tbc/pym/db_mapping.py | 14 +++++++- tbc/pym/jobs.py | 20 ++--------- tbc/pym/package.py | 2 +- tbc/pym/readconf.py | 8 ++--- tbc/pym/sqlquerys.py | 35 ++++++++++++++----- tbc/pym/sync.py | 44 ++++++++++++++++++++---- tbc/pym/updatedb.py | 2 +- 10 files changed, 153 insertions(+), 104 deletions(-) diff --git a/sql/structure_dump.sql b/sql/structure_dump.sql index 5fe548b..9d9ecba 100644 --- a/sql/structure_dump.sql +++ b/sql/structure_dump.sql @@ -3,7 +3,7 @@ -- http://www.phpmyadmin.net -- -- Host: localhost --- Generation Time: Apr 18, 2015 at 08:31 PM +-- Generation Time: May 09, 2015 at 02:34 PM -- Server version: 10.0.15-MariaDB-log -- PHP Version: 5.6.5-pl0-gentoo @@ -17,7 +17,7 @@ SET time_zone = "+00:00"; /*!40101 SET NAMES utf8 */; -- --- Database: `tbc` +-- Database: `zobcs` -- DELIMITER $$ @@ -57,10 +57,10 @@ CREATE TABLE IF NOT EXISTS `build_jobs` ( `setup_id` int(11) NOT NULL, `config_id` int(11) NOT NULL, `status` enum('Waiting','Building','Looked') NOT NULL DEFAULT 'Waiting', - `build_now` tinyint(1) NOT NULL DEFAULT '0', + `build_now` tinyint(1) NOT NULL, `removebin` tinyint(1) NOT NULL, `time_stamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP -) ENGINE=MyISAM AUTO_INCREMENT=1325 DEFAULT CHARSET=utf8 COMMENT='The build work list'; +) ENGINE=MyISAM AUTO_INCREMENT=3113 DEFAULT CHARSET=utf8 COMMENT='The build work list'; -- -------------------------------------------------------- @@ -99,7 +99,7 @@ CREATE TABLE IF NOT EXISTS `build_jobs_use` ( `build_job_id` int(11) NOT NULL, `use_id` int(11) NOT NULL, `status` tinyint(1) NOT NULL DEFAULT '0' -) ENGINE=MyISAM AUTO_INCREMENT=6067 DEFAULT CHARSET=utf8; +) ENGINE=MyISAM AUTO_INCREMENT=17343 DEFAULT CHARSET=utf8; -- -------------------------------------------------------- @@ -115,7 +115,7 @@ CREATE TABLE IF NOT EXISTS `build_logs` ( `log_hash` varchar(100) NOT NULL, `bug_id` int(10) NOT NULL DEFAULT '0', `time_stamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP -) ENGINE=MyISAM AUTO_INCREMENT=15154 DEFAULT CHARSET=utf8 COMMENT='Main log info for the builds'; +) ENGINE=MyISAM AUTO_INCREMENT=17625 DEFAULT CHARSET=utf8 COMMENT='Main log info for the builds'; -- -------------------------------------------------------- @@ -130,7 +130,7 @@ CREATE TABLE IF NOT EXISTS `build_logs_config` ( `einfo_id` int(11) NOT NULL, `logname` varchar(150) NOT NULL COMMENT 'filename of the log', `time_stamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP -) ENGINE=MyISAM AUTO_INCREMENT=41689 DEFAULT CHARSET=utf8; +) ENGINE=MyISAM AUTO_INCREMENT=44160 DEFAULT CHARSET=utf8; -- -------------------------------------------------------- @@ -154,7 +154,7 @@ CREATE TABLE IF NOT EXISTS `build_logs_errors` ( `id` int(11) NOT NULL, `build_log_id` int(11) NOT NULL, `error_id` int(11) NOT NULL -) ENGINE=MyISAM AUTO_INCREMENT=2616 DEFAULT CHARSET=utf8; +) ENGINE=MyISAM AUTO_INCREMENT=3379 DEFAULT CHARSET=utf8; -- -------------------------------------------------------- @@ -168,7 +168,7 @@ CREATE TABLE IF NOT EXISTS `build_logs_hilight` ( `start_line` int(11) NOT NULL, `end_line` int(11) NOT NULL, `hilight_css_id` int(11) NOT NULL -) ENGINE=MyISAM AUTO_INCREMENT=350435 DEFAULT CHARSET=utf8; +) ENGINE=MyISAM AUTO_INCREMENT=413189 DEFAULT CHARSET=utf8; -- -------------------------------------------------------- @@ -190,9 +190,9 @@ CREATE TABLE IF NOT EXISTS `build_logs_qa` ( CREATE TABLE IF NOT EXISTS `build_logs_repoman` ( `id` int(11) NOT NULL, - `build_logs_id` int(11) NOT NULL, + `build_log_id` int(11) NOT NULL, `summery_text` text NOT NULL -) ENGINE=MyISAM DEFAULT CHARSET=utf8; +) ENGINE=MyISAM AUTO_INCREMENT=51 DEFAULT CHARSET=utf8; -- -------------------------------------------------------- @@ -205,7 +205,7 @@ CREATE TABLE IF NOT EXISTS `build_logs_use` ( `build_log_id` int(11) NOT NULL, `use_id` int(11) NOT NULL, `status` tinyint(1) NOT NULL -) ENGINE=MyISAM AUTO_INCREMENT=66200 DEFAULT CHARSET=utf8; +) ENGINE=MyISAM AUTO_INCREMENT=76204 DEFAULT CHARSET=utf8; -- -------------------------------------------------------- @@ -281,7 +281,7 @@ CREATE TABLE IF NOT EXISTS `ebuilds` ( `checksum` varchar(100) NOT NULL, `active` tinyint(1) NOT NULL DEFAULT '0', `time_stamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP -) ENGINE=MyISAM AUTO_INCREMENT=78536 DEFAULT CHARSET=utf8 COMMENT='Version main table (V)'; +) ENGINE=MyISAM AUTO_INCREMENT=81657 DEFAULT CHARSET=utf8 COMMENT='Version main table (V)'; -- -------------------------------------------------------- @@ -294,7 +294,7 @@ CREATE TABLE IF NOT EXISTS `ebuilds_iuse` ( `ebuild_id` int(11) NOT NULL, `use_id` int(11) NOT NULL, `status` tinyint(1) NOT NULL -) ENGINE=MyISAM AUTO_INCREMENT=427416 DEFAULT CHARSET=utf8; +) ENGINE=MyISAM AUTO_INCREMENT=456109 DEFAULT CHARSET=utf8; -- -------------------------------------------------------- @@ -307,7 +307,7 @@ CREATE TABLE IF NOT EXISTS `ebuilds_keywords` ( `ebuild_id` int(11) NOT NULL, `keyword_id` int(11) NOT NULL, `status` enum('Stable','Unstable','Negative') NOT NULL -) ENGINE=MyISAM AUTO_INCREMENT=532825 DEFAULT CHARSET=utf8; +) ENGINE=MyISAM AUTO_INCREMENT=559659 DEFAULT CHARSET=utf8; -- -------------------------------------------------------- @@ -319,7 +319,7 @@ CREATE TABLE IF NOT EXISTS `ebuilds_metadata` ( `id` int(11) NOT NULL, `ebuild_id` int(11) NOT NULL, `revision` varchar(10) NOT NULL COMMENT 'CVS revision' -) ENGINE=MyISAM AUTO_INCREMENT=78536 DEFAULT CHARSET=utf8; +) ENGINE=MyISAM AUTO_INCREMENT=81657 DEFAULT CHARSET=utf8; -- -------------------------------------------------------- @@ -331,7 +331,7 @@ CREATE TABLE IF NOT EXISTS `ebuilds_restrictions` ( `id` int(11) NOT NULL, `ebuild_id` int(11) NOT NULL, `restriction_id` int(11) NOT NULL -) ENGINE=MyISAM AUTO_INCREMENT=13453 DEFAULT CHARSET=utf8; +) ENGINE=MyISAM AUTO_INCREMENT=14328 DEFAULT CHARSET=utf8; -- -------------------------------------------------------- @@ -342,7 +342,7 @@ CREATE TABLE IF NOT EXISTS `ebuilds_restrictions` ( CREATE TABLE IF NOT EXISTS `emails` ( `email_id` int(11) NOT NULL, `email` varchar(150) NOT NULL -) ENGINE=MyISAM AUTO_INCREMENT=675 DEFAULT CHARSET=utf8; +) ENGINE=MyISAM AUTO_INCREMENT=679 DEFAULT CHARSET=utf8; -- -------------------------------------------------------- @@ -353,7 +353,7 @@ CREATE TABLE IF NOT EXISTS `emails` ( CREATE TABLE IF NOT EXISTS `emerge_info` ( `einfo_id` int(11) NOT NULL, `emerge_info_text` text NOT NULL -) ENGINE=MyISAM AUTO_INCREMENT=16164 DEFAULT CHARSET=utf8; +) ENGINE=MyISAM AUTO_INCREMENT=18635 DEFAULT CHARSET=utf8; -- -------------------------------------------------------- @@ -445,7 +445,7 @@ CREATE TABLE IF NOT EXISTS `logs` ( `log_type` enum('info','error','debug','qa') NOT NULL, `msg` text NOT NULL, `time_stamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP -) ENGINE=MyISAM AUTO_INCREMENT=9536253 DEFAULT CHARSET=utf8; +) ENGINE=MyISAM AUTO_INCREMENT=22201403 DEFAULT CHARSET=utf8; -- -------------------------------------------------------- @@ -461,7 +461,7 @@ CREATE TABLE IF NOT EXISTS `packages` ( `checksum` varchar(100) NOT NULL, `active` tinyint(1) NOT NULL, `time_stamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP -) ENGINE=MyISAM AUTO_INCREMENT=18476 DEFAULT CHARSET=utf8 COMMENT='Packages main table (P)'; +) ENGINE=MyISAM AUTO_INCREMENT=18514 DEFAULT CHARSET=utf8 COMMENT='Packages main table (P)'; -- -------------------------------------------------------- @@ -473,7 +473,7 @@ CREATE TABLE IF NOT EXISTS `packages_emails` ( `id` int(11) NOT NULL, `package_id` int(11) NOT NULL, `email_id` int(11) NOT NULL -) ENGINE=MyISAM AUTO_INCREMENT=22297 DEFAULT CHARSET=utf8; +) ENGINE=MyISAM AUTO_INCREMENT=22365 DEFAULT CHARSET=utf8; -- -------------------------------------------------------- @@ -485,7 +485,7 @@ CREATE TABLE IF NOT EXISTS `packages_metadata` ( `id` int(11) NOT NULL, `package_id` int(11) NOT NULL, `checksum` varchar(100) NOT NULL -) ENGINE=MyISAM AUTO_INCREMENT=18475 DEFAULT CHARSET=utf8; +) ENGINE=MyISAM AUTO_INCREMENT=18513 DEFAULT CHARSET=utf8; -- -------------------------------------------------------- @@ -517,7 +517,8 @@ CREATE TABLE IF NOT EXISTS `restrictions` ( CREATE TABLE IF NOT EXISTS `setups` ( `setup_id` int(11) NOT NULL, - `setup` varchar(100) NOT NULL + `setup` varchar(100) NOT NULL, + `profile` varchar(150) NOT NULL ) ENGINE=MyISAM AUTO_INCREMENT=4 DEFAULT CHARSET=utf8; -- -------------------------------------------------------- @@ -529,7 +530,7 @@ CREATE TABLE IF NOT EXISTS `setups` ( CREATE TABLE IF NOT EXISTS `uses` ( `use_id` int(11) NOT NULL, `flag` varchar(50) NOT NULL -) ENGINE=MyISAM AUTO_INCREMENT=6153 DEFAULT CHARSET=utf8 COMMENT='Use flags main table'; +) ENGINE=MyISAM AUTO_INCREMENT=6180 DEFAULT CHARSET=utf8 COMMENT='Use flags main table'; -- -- Indexes for dumped tables @@ -599,7 +600,7 @@ ALTER TABLE `build_logs_qa` -- Indexes for table `build_logs_repoman` -- ALTER TABLE `build_logs_repoman` - ADD PRIMARY KEY (`id`), ADD KEY `build_logs_id` (`build_logs_id`); + ADD PRIMARY KEY (`id`), ADD KEY `build_logs_id` (`build_log_id`); -- -- Indexes for table `build_logs_use` @@ -765,7 +766,7 @@ ALTER TABLE `uses` -- AUTO_INCREMENT for table `build_jobs` -- ALTER TABLE `build_jobs` -MODIFY `build_job_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=1325; +MODIFY `build_job_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=3113; -- -- AUTO_INCREMENT for table `build_jobs_emerge_options` -- @@ -780,17 +781,17 @@ MODIFY `id` int(11) NOT NULL AUTO_INCREMENT; -- AUTO_INCREMENT for table `build_jobs_use` -- ALTER TABLE `build_jobs_use` -MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=6067; +MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=17343; -- -- AUTO_INCREMENT for table `build_logs` -- ALTER TABLE `build_logs` -MODIFY `build_log_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=15154; +MODIFY `build_log_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=17625; -- -- AUTO_INCREMENT for table `build_logs_config` -- ALTER TABLE `build_logs_config` -MODIFY `log_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=41689; +MODIFY `log_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=44160; -- -- AUTO_INCREMENT for table `build_logs_emerge_options` -- @@ -800,12 +801,12 @@ MODIFY `id` int(11) NOT NULL AUTO_INCREMENT; -- AUTO_INCREMENT for table `build_logs_errors` -- ALTER TABLE `build_logs_errors` -MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=2616; +MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=3379; -- -- AUTO_INCREMENT for table `build_logs_hilight` -- ALTER TABLE `build_logs_hilight` -MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=350435; +MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=413189; -- -- AUTO_INCREMENT for table `build_logs_qa` -- @@ -815,12 +816,12 @@ MODIFY `id` int(11) NOT NULL AUTO_INCREMENT; -- AUTO_INCREMENT for table `build_logs_repoman` -- ALTER TABLE `build_logs_repoman` -MODIFY `id` int(11) NOT NULL AUTO_INCREMENT; +MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=51; -- -- AUTO_INCREMENT for table `build_logs_use` -- ALTER TABLE `build_logs_use` -MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=66200; +MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=76204; -- -- AUTO_INCREMENT for table `categories` -- @@ -845,37 +846,37 @@ MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=7; -- AUTO_INCREMENT for table `ebuilds` -- ALTER TABLE `ebuilds` -MODIFY `ebuild_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=78536; +MODIFY `ebuild_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=81657; -- -- AUTO_INCREMENT for table `ebuilds_iuse` -- ALTER TABLE `ebuilds_iuse` -MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=427416; +MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=456109; -- -- AUTO_INCREMENT for table `ebuilds_keywords` -- ALTER TABLE `ebuilds_keywords` -MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=532825; +MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=559659; -- -- AUTO_INCREMENT for table `ebuilds_metadata` -- ALTER TABLE `ebuilds_metadata` -MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=78536; +MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=81657; -- -- AUTO_INCREMENT for table `ebuilds_restrictions` -- ALTER TABLE `ebuilds_restrictions` -MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=13453; +MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=14328; -- -- AUTO_INCREMENT for table `emails` -- ALTER TABLE `emails` -MODIFY `email_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=675; +MODIFY `email_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=679; -- -- AUTO_INCREMENT for table `emerge_info` -- ALTER TABLE `emerge_info` -MODIFY `einfo_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=16164; +MODIFY `einfo_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=18635; -- -- AUTO_INCREMENT for table `emerge_options` -- @@ -910,22 +911,22 @@ MODIFY `keyword_id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'keyword index',AUTO -- AUTO_INCREMENT for table `logs` -- ALTER TABLE `logs` -MODIFY `log_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=9536253; +MODIFY `log_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=22201403; -- -- AUTO_INCREMENT for table `packages` -- ALTER TABLE `packages` -MODIFY `package_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=18476; +MODIFY `package_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=18514; -- -- AUTO_INCREMENT for table `packages_emails` -- ALTER TABLE `packages_emails` -MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=22297; +MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=22365; -- -- AUTO_INCREMENT for table `packages_metadata` -- ALTER TABLE `packages_metadata` -MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=18475; +MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=18513; -- -- AUTO_INCREMENT for table `repos` -- @@ -945,7 +946,7 @@ MODIFY `setup_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=4; -- AUTO_INCREMENT for table `uses` -- ALTER TABLE `uses` -MODIFY `use_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=6153; +MODIFY `use_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=6180; DELIMITER $$ -- -- Events diff --git a/tbc/pym/build_log.py b/tbc/pym/build_log.py index a7d5314..bfbe8ec 100644 --- a/tbc/pym/build_log.py +++ b/tbc/pym/build_log.py @@ -23,16 +23,16 @@ portage.proxy.lazyimport.lazyimport(globals(), from tbc.repoman_tbc import tbc_repoman from tbc.text import get_log_text_dict from tbc.package import tbc_package -from tbc.readconf import get_conf_settings +from zobcs.readconf import read_config_settings from tbc.flags import tbc_use_flags from tbc.ConnectionManager import NewConnection from tbc.sqlquerys import add_tbc_logs, get_config_id, get_ebuild_id_db, add_new_buildlog, \ get_package_info, get_build_job_id, get_use_id, get_config_info, get_hilight_info, get_error_info_list, \ add_e_info, get_fail_times, add_fail_times, update_fail_times, del_old_build_jobs, add_old_ebuild, \ - update_buildjobs_status + update_buildjobs_status, update_manifest_sql, add_repoman_qa from sqlalchemy.orm import sessionmaker -def get_build_dict_db(session, config_id, settings, pkg): +def get_build_dict_db(session, config_id, settings, tbc_settings_dict, pkg): myportdb = portage.portdbapi(mysettings=settings) cpvr_list = catpkgsplit(pkg.cpv, silent=1) categories = cpvr_list[0] @@ -77,10 +77,10 @@ def get_build_dict_db(session, config_id, settings, pkg): log_msg = "%s:%s Don't have any ebuild_id!" % (pkg.cpv, repo,) add_tbc_logs(session, log_msg, "info", config_id) update_manifest_sql(session, build_dict['package_id'], "0") - init_package = tbc_package(session, settings, myportdb) + init_package = tbc_package(session, settings, myportdb, config_id, tbc_settings_dict) init_package.update_package_db(build_dict['package_id']) ebuild_id_list, status = get_ebuild_id_db(session, build_dict['checksum'], build_dict['package_id']) - if status and ebuild_id is None: + if status and ebuild_id_list is None: log_msg = "%s:%s Don't have any ebuild_id!" % (pkg.cpv, repo,) add_tbc_logs(session, log_msg, "error", config_id) else: @@ -209,8 +209,12 @@ def get_buildlog_info(session, settings, pkg, build_dict): repoman_error_list = init_repoman.check_repoman(build_dict['cpv'], pkg.repo) if repoman_error_list != []: sum_build_log_list.append("1") # repoman = 1 + else: + repoman_error_list = False if qa_error_list != []: sum_build_log_list.append("2") # qa = 2 + else: + qa_error_list = False error_search_line = "^ \\* ERROR: " for error_log_line in error_log_list: if re.search(error_search_line, error_log_line): @@ -234,8 +238,7 @@ def get_emerge_info_id(settings, trees, session, config_id): return "\n".join(emerge_info_list) def add_buildlog_main(settings, pkg, trees): - reader=get_conf_settings() - tbc_settings_dict=reader.read_tbc_settings_all() + tbc_settings_dict = read_config_settings() config = tbc_settings_dict['tbc_config'] hostname =tbc_settings_dict['hostname'] host_config = hostname + "/" + config @@ -245,7 +248,7 @@ def add_buildlog_main(settings, pkg, trees): if pkg.type_name == "binary": build_dict = None else: - build_dict = get_build_dict_db(session, config_id, settings, pkg) + build_dict = get_build_dict_db(session, config_id, settings, tbc_settings_dict, pkg) if build_dict is None: log_msg = "Package %s:%s is NOT logged." % (pkg.cpv, pkg.repo,) add_tbc_logs(session, log_msg, "info", config_id) @@ -273,6 +276,7 @@ def add_buildlog_main(settings, pkg, trees): log_msg = "Package %s:%s is NOT logged." % (pkg.cpv, pkg.repo,) add_tbc_logs(session, log_msg, "info", config_id) else: + add_repoman_qa(session, build_log_dict, log_id) os.chmod(settings.get("PORTAGE_LOG_FILE"), 0o664) log_msg = "Package: %s:%s is logged." % (pkg.cpv, pkg.repo,) add_tbc_logs(session, log_msg, "info", config_id) @@ -291,13 +295,9 @@ def log_fail_queru(session, build_dict, settings): else: build_log_dict = {} error_log_list = [] - qa_error_list = [] - repoman_error_list = [] sum_build_log_list = [] sum_build_log_list.append("2") error_log_list.append(build_dict['type_fail']) - build_log_dict['repoman_error_list'] = repoman_error_list - build_log_dict['qa_error_list'] = qa_error_list build_log_dict['summary_error_list'] = sum_build_log_list if build_dict['type_fail'] == 'merge fail': error_log_list = [] diff --git a/tbc/pym/check_setup.py b/tbc/pym/check_setup.py index 46d4391..afd6580 100644 --- a/tbc/pym/check_setup.py +++ b/tbc/pym/check_setup.py @@ -9,16 +9,19 @@ import errno from portage.exception import DigestException, FileNotFound, ParseError, PermissionDenied from tbc.text import get_file_text from tbc.sqlquerys import get_config_all_info, add_tbc_logs, get_configmetadata_info, get_setup_info +from tbc.sync import git_pull def check_make_conf(session, config_id, tbc_settings_dict): log_msg = "Checking configs for changes and errors" add_tbc_logs(session, log_msg, "info", config_id) + git_repo = tbc_settings_dict['tbc_gitrepopath'] + "/" + git_pull(session, git_repo, config_id) configsDict = {} for ConfigInfo in get_config_all_info(session): attDict={} # Set the config dir SetupInfo = get_setup_info(session, ConfigInfo.ConfigId) - check_config_dir = "/var/cache/tbc/" + tbc_settings_dict['tbc_gitreponame'] + "/" + ConfigInfo.Hostname +"/" + SetupInfo.Setup + "/" + check_config_dir = tbc_settings_dict['tbc_gitrepopath'] + "/" + ConfigInfo.Hostname +"/" + SetupInfo.Setup + "/" make_conf_file = check_config_dir + "etc/portage/make.conf" ConfigsMetaDataInfo = get_configmetadata_info(session, ConfigInfo.ConfigId) # Check if we can take a checksum on it. @@ -48,7 +51,9 @@ def check_make_conf(session, config_id, tbc_settings_dict): log_msg = "Checking configs for changes and errors ... Done" add_tbc_logs(session, log_msg, "info", config_id) -def check_make_conf_guest(session, config_id): +def check_make_conf_guest(session, tbc_settings_dict, config_id): + git_repo = tbc_settings_dict['tbc_gitrepopath'] + "/" + git_pull(session, git_repo, config_id) make_conf_file = "/etc/portage/make.conf" # Check if we can open the file and close it # Check if we have some error in the file (portage.util.getconfig) @@ -68,7 +73,7 @@ def check_make_conf_guest(session, config_id): return False return True -def check_configure_guest(session, config_id): - pass_make_conf = check_make_conf_guest(session, config_id) +def check_configure_guest(session, tbc_settings_dict, config_id): + pass_make_conf = check_make_conf_guest(session, tbc_settings_dict, config_id) print(pass_make_conf) - return pass_make_conf \ No newline at end of file + return pass_make_conf diff --git a/tbc/pym/db_mapping.py b/tbc/pym/db_mapping.py index c215456..31e1b66 100644 --- a/tbc/pym/db_mapping.py +++ b/tbc/pym/db_mapping.py @@ -16,6 +16,7 @@ class Keywords(Base): class Setups(Base): SetupId = Column('setup_id', Integer, primary_key=True) Setup = Column('setup', String(100)) + Profile = Column('profile', String(150)) __tablename__ = 'setups' class Configs(Base): @@ -46,7 +47,6 @@ class Jobs(Base): class ConfigsMetaData(Base): Id = Column('id', Integer, primary_key=True) ConfigId = Column('config_id', Integer, ForeignKey('configs.config_id')) - Profile = Column('profile', String(150)) KeywordId = Column('keyword_id', Integer, ForeignKey('keywords.keyword_id')) MakeConfText = Column('make_conf_text', Text) Checksum = Column('checksum', String(100)) @@ -216,6 +216,18 @@ class BuildLogsUse(Base): Status = Column('status', Boolean, default=False) __tablename__ = 'build_logs_use' +class BuildLogsQA(Base): + Id = Column('id', Integer, primary_key=True) + BuildLogId = Column('build_log_id', Integer, ForeignKey('build_logs.build_log_id')) + SummeryText = Column('summery_text', Text) + __tablename__ = 'build_logs_qa' + +class BuildLogsRepoman(Base): + Id = Column('id', Integer, primary_key=True) + BuildLogId = Column('build_log_id', Integer, ForeignKey('build_logs.build_log_id')) + SummeryText = Column('summery_text', Text) + __tablename__ = 'build_logs_repoman' + class ErrorsInfo(Base): ErrorId = Column('error_id', Integer, primary_key=True) ErrorName = Column('error_name', String) diff --git a/tbc/pym/jobs.py b/tbc/pym/jobs.py index 840932e..8559cf5 100644 --- a/tbc/pym/jobs.py +++ b/tbc/pym/jobs.py @@ -43,26 +43,11 @@ def jobs_main(session, config_id): # update_job_list(session, "Fail", job_id) # log_msg = "Job %s did fail." % (job_id,) # add_tbc_logs(session, log_msg, "info", config_id) - elif job == "gsync": - update_job_list(session, "Runing", job_id) - log_msg = "Job %s is runing." % (job_id,) - add_tbc_logs(session, log_msg, "info", config_id) - result = True - # result = git_pull(session) - if result: - update_job_list(session, "Done", job_id) - log_msg = "Job %s is done.." % (job_id,) - add_tbc_logs(session, log_msg, "info", config_id) - else: - update_job_list(session, "Fail", job_id) - log_msg = "Job %s did fail." % (job_id,) - add_tbc_logs(session, log_msg, "info", config_id) elif job == "esync": update_job_list(session, "Runing", job_id) log_msg = "Job %s is runing." % (job_id,) add_tbc_logs(session, log_msg, "info", config_id) - result = sync_tree(session) - if result: + if sync_tree(session) and update_db_main(session, config_id): update_job_list(session, "Done", job_id) log_msg = "Job %s is done.." % (job_id,) add_tbc_logs(session, log_msg, "info", config_id) @@ -74,8 +59,7 @@ def jobs_main(session, config_id): update_job_list(session, "Runing", job_id) log_msg = "Job %s is runing." % (job_id,) add_tbc_logs(session, log_msg, "info", config_id) - result = update_db_main(session, config_id) - if result: + ifupdate_db_main(session, config_id): update_job_list(session, "Done", job_id) log_msg = "Job %s is done.." % (job_id,) add_tbc_logs(session, log_msg, "info", config_id) diff --git a/tbc/pym/package.py b/tbc/pym/package.py index af1bff5..5f172cb 100644 --- a/tbc/pym/package.py +++ b/tbc/pym/package.py @@ -26,7 +26,7 @@ class tbc_package(object): def change_config(self, host_config): # Change config_root config_setup = table config - my_new_setup = "/var/cache/tbc/" + self._tbc_settings_dict['tbc_gitreponame'] + "/" + host_config + "/" + my_new_setup = tbc_settings_dict['tbc_gitrepopath'] + "/" + host_config + "/" mysettings_setup = portage.config(config_root = my_new_setup) return mysettings_setup diff --git a/tbc/pym/readconf.py b/tbc/pym/readconf.py index db1056e..39131db 100644 --- a/tbc/pym/readconf.py +++ b/tbc/pym/readconf.py @@ -32,10 +32,10 @@ class get_conf_settings(object): if element[0] == 'SQLPASSWD': # Password get_sql_passwd = element[1] # Buildhost root (dir for host/setup on host) - if element[0] == 'ZOBCSGITREPONAME': - get_tbc_gitreponame = element[1] + if element[0] == 'TBCREPOPATH': + get_tbc_gitrepopath = element[1] # Buildhost setup (host/setup on guest) - if element[0] == 'ZOBCSCONFIG': + if element[0] == 'TBCCONFIG': get_tbc_config = element[1] # if element[0] == 'LOGFILE': # get_tbc_logfile = element[1] @@ -47,7 +47,7 @@ class get_conf_settings(object): tbc_settings_dict['sql_host'] = get_sql_host.rstrip('\n') tbc_settings_dict['sql_user'] = get_sql_user.rstrip('\n') tbc_settings_dict['sql_passwd'] = get_sql_passwd.rstrip('\n') - tbc_settings_dict['tbc_gitreponame'] = get_tbc_gitreponame.rstrip('\n') + tbc_settings_dict['tbc_gitrepopath'] = get_tbc_gitrepopath.rstrip('\n') tbc_settings_dict['tbc_config'] = get_tbc_config.rstrip('\n') tbc_settings_dict['hostname'] = getfqdn() # tbc_settings_dict['tbc_logfile'] = get_tbc_logfile.rstrip('\n') diff --git a/tbc/pym/sqlquerys.py b/tbc/pym/sqlquerys.py index 623f0b0..f9fc8b4 100644 --- a/tbc/pym/sqlquerys.py +++ b/tbc/pym/sqlquerys.py @@ -7,7 +7,7 @@ from tbc.db_mapping import Configs, Logs, ConfigsMetaData, Jobs, BuildJobs, Pack Uses, ConfigsEmergeOptions, EmergeOptions, HiLight, BuildLogs, BuildLogsConfig, BuildJobsUse, BuildJobsRedo, \ HiLightCss, BuildLogsHiLight, BuildLogsEmergeOptions, BuildLogsErrors, ErrorsInfo, EmergeInfo, BuildLogsUse, \ BuildJobsEmergeOptions, EbuildsMetadata, EbuildsIUse, Restrictions, EbuildsRestrictions, EbuildsKeywords, \ - Keywords, PackagesMetadata, Emails, PackagesEmails, Setups + Keywords, PackagesMetadata, Emails, PackagesEmails, Setups, BuildLogsQA, BuildLogsRepoman from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound from sqlalchemy import and_, or_ @@ -69,15 +69,17 @@ def is_build_job_done(session, build_job_id): def get_packages_to_build(session, config_id): SetupInfo = get_setup_info(session, config_id) BuildJobsTmp = session.query(BuildJobs).filter(BuildJobs.SetupId==SetupInfo.SetupId). \ - order_by(BuildJobs.BuildJobId) - if session.query(BuildJobs).filter_by(SetupId = SetupInfo.SetupId).filter_by(BuildNow = True).filter_by(Status = 'Waiting').all() == [] and session.query(BuildJobs).filter_by(SetupId = SetupInfo.SetupId).filter_by(Status = 'Waiting').all() == []: + order_by(BuildJobs.BuildJobId).filter_by(Status = 'Waiting') + if BuildJobsTmp.all() == []: return None - if not BuildJobsTmp.filter_by(BuildNow = True).first() is None: - BuildJobsInfo = session.query(BuildJobs).filter_by(SetupId = SetupInfo.SetupId).filter_by(BuildNow = True). \ - filter_by(Status = 'Waiting').order_by(BuildJobs.BuildJobId).first() + elif BuildJobsTmp.filter_by(BuildNow = True).all() != []: + BuildJobsInfo = BuildJobsTmp.filter_by(BuildNow = True).first() + elif BuildJobsTmp.filter_by(BuildNow = False).all() != []: + BuildJobsInfo = BuildJobsTmp.filter_by(BuildNow = False).first() else: - BuildJobsInfo = session.query(BuildJobs).filter_by(SetupId = SetupInfo.SetupId).filter_by(Status = 'Waiting').\ - order_by(BuildJobs.BuildJobId).first() + log_msg = "BuildJobsTmp found job but the if state mant did not." + add_zobcs_logs(session, log_msg, "error", config_id) + return None update_buildjobs_status(session, BuildJobsInfo.BuildJobId, 'Looked', config_id) EbuildsInfo = session.query(Ebuilds).filter_by(EbuildId = BuildJobsInfo.EbuildId).one() PackagesInfo, CategoriesInfo = session.query(Packages, Categories).filter(Packages.PackageId==EbuildsInfo.PackageId).filter(Packages.CategoryId==Categories.CategoryId).one() @@ -266,6 +268,22 @@ def add_new_buildlog(session, build_dict, build_log_dict): del_old_build_jobs(session, build_dict['build_job_id']) return build_log_id +def add_repoman_qa(session, build_log_dict, log_id): + repoman_error = "" + qa_error = "" + if build_log_dict['qa_error_list']: + for qa_text in build_log_dict['qa_error_list']: + qa_error = qa_error + build_log_dict['qa_error_list'] + NewBuildLogQA = BuildLogsQA(BuildLogId = log_id, SummeryText = qa_error) + session.add(NewBuildLogQA) + session.commit() + if build_log_dict['repoman_error_list']: + for repoman_text in build_log_dict['repoman_error_list']: + repoman_error = repoman_error + repoman_text + NewBuildLogRepoman = BuildLogsRepoman(BuildLogId = log_id, SummeryText = repoman_error) + session.add(NewBuildLogRepoman) + session.commit() + def update_fail_times(session, FailInfo): NewBuildJobs = session.query(BuildJobs).filter_by(BuildJobId = FailInfo.BuildJobId).one() NewBuildJobs.TimeStamp = datetime.datetime.utcnow() @@ -279,7 +297,6 @@ def get_fail_times(session, build_dict): return True def add_fail_times(session, fail_querue_dict): - print(fail_querue_dict) NewBuildJobsRedo = BuildJobsRedo(BuildJobId = fail_querue_dict['build_job_id'], FailType = fail_querue_dict['fail_type'], FailTimes = fail_querue_dict['fail_times']) session.add(NewBuildJobsRedo) session.commit() diff --git a/tbc/pym/sync.py b/tbc/pym/sync.py index a75562f..f1bc56a 100644 --- a/tbc/pym/sync.py +++ b/tbc/pym/sync.py @@ -7,9 +7,10 @@ import os import errno import sys import time +from pygit2 import Repository, GIT_MERGE_ANALYSIS_FASTFORWARD, GIT_MERGE_ANALYSIS_NORMAL, \ + GIT_MERGE_ANALYSIS_UP_TO_DATE from _emerge.main import emerge_main -from tbc.readconf import get_conf_settings from tbc.sqlquerys import get_config_id, add_tbc_logs, get_config_all_info, get_configmetadata_info from tbc.updatedb import update_db_main from tbc.readconf import read_config_settings @@ -20,7 +21,7 @@ def sync_tree(session): _config = tbc_settings_dict['tbc_config'] config_id = get_config_id(session, _config, _hostname) host_config = _hostname +"/" + _config - default_config_root = "/var/cache/tbc/" + tbc_settings_dict['tbc_gitreponame'] + "/" + host_config + "/" + default_config_root = tbc_settings_dict['tbc_gitrepopath'] + "/" + host_config + "/" mysettings = portage.config(config_root = default_config_root) GuestBusy = True log_msg = "Waiting for Guest to be idel" @@ -65,9 +66,38 @@ def sync_tree(session): pass log_msg = "Emerge --sync ... Done." add_tbc_logs(session, log_msg, "info", config_id) - result = update_db_main(session, config_id) - if result: - return True + return True + +def git_pull(session, git_repo, config_id): + log_msg = "Git pull" + add_zobcs_logs(session, log_msg, "info", config_id) + repo = Repository(git_repo + ".git") + remote = repo.remotes["origin"] + remote.fetch() + remote_master_id = repo.lookup_reference('refs/remotes/origin/master').target + merge_result, _ = repo.merge_analysis(remote_master_id) + if merge_result & GIT_MERGE_ANALYSIS_UP_TO_DATE: + log_msg = "Repo is up to date" + add_zobcs_logs(session, log_msg, "info", config_id) + elif merge_result & GIT_MERGE_ANALYSIS_FASTFORWARD: + repo.checkout_tree(repo.get(remote_master_id)) + master_ref = repo.lookup_reference('refs/heads/master') + master_ref.set_target(remote_master_id) + repo.head.set_target(remote_master_id) + elif merge_result & GIT_MERGE_ANALYSIS_NORMAL: + repo.merge(remote_master_id) + assert repo.index.conflicts is None, 'Conflicts, ahhhh!' + user = repo.default_signature + tree = repo.index.write_tree() + commit = repo.create_commit('HEAD', + user, + user, + 'Merge!', + tree, + [repo.head.target, remote_master_id]) + repo.state_cleanup() else: - log_msg = "Updatedb fail" - add_tbc_logs(session, log_msg, "info", config_id) + raise AssertionError('Unknown merge analysis result') + log_msg = "Git pull ... Done" + add_zobcs_logs(session, log_msg, "info", config_id) + return True diff --git a/tbc/pym/updatedb.py b/tbc/pym/updatedb.py index 12b94c5..90a9e0a 100644 --- a/tbc/pym/updatedb.py +++ b/tbc/pym/updatedb.py @@ -25,7 +25,7 @@ def init_portage_settings(session, config_id, tbc_settings_dict): # Get default config from the configs table and default_config=1 host_config = tbc_settings_dict['hostname'] +"/" + tbc_settings_dict['tbc_config'] - default_config_root = "/var/cache/tbc/" + tbc_settings_dict['tbc_gitreponame'] + "/" + host_config + "/" + default_config_root = tbc_settings_dict['tbc_gitrepopath'] + "/" + host_config + "/" # Set config_root (PORTAGE_CONFIGROOT) to default_config_root mysettings = portage.config(config_root = default_config_root)
