Please hold off on this version, there is an error in the layers set()
handling. I will send a v2.
On Fri, Oct 27, 2023 at 3:19 PM Tim Orling via lists.yoctoproject.org
<ticotimo=gmail....@lists.yoctoproject.org> wrote:
> This is a handy helper script to query the Buildbot REST API to
> create a list of Yocto Project Compatible 2.0 layers, as defined by
> successful running of the "check-layers*" jobs.
>
> The script only considers the 'jobs' ("Builders" in Buildbot speak) you
> ask it to query.
>
> The script looks at the latest -n successful builds for a branch that
> matches from a list. The number of builds approach was used because
> the property_list cannot be queried for "branch" in the "builders" API.
>
> It looks for a "step" name pattern via a regex, with which all currently
> running "check-layer" and "check-layer-nightly" jobs conform.
>
> Once a branch has successfully been queried, it is removed from the list
> in further iterations. The intent is to only find the latest passing
> list of compatible layers, for each branch.
>
> The output is a yp_compatible_layers.json file which can
> be pretty printed with:
>
> cat yp_compatible_layers.json | jq
>
> Signed-off-by: Tim Orling <tim.orl...@konsulko.com>
> ---
> scripts/list-yp-compatible-layers.py | 155 +++++++++++++++++++++++++++
> 1 file changed, 155 insertions(+)
> create mode 100755 scripts/list-yp-compatible-layers.py
>
> diff --git a/scripts/list-yp-compatible-layers.py
> b/scripts/list-yp-compatible-layers.py
> new file mode 100755
> index 0000000..f247057
> --- /dev/null
> +++ b/scripts/list-yp-compatible-layers.py
> @@ -0,0 +1,155 @@
> +#!/usr/bin/env python3
> +
> +# Query layers that are Yocto Project Compatible (2.0) on the AutoBuilder
> +# and make a yp_compatible_layers.json file as an output
> +#
> +# Copyright (C) 2023 Konsulko Group
> +# Author: Tim Orling <tim.orl...@konsulko.com>
> +#
> +# Licensed under the MIT license, see COPYING.MIT for details
> +#
> +# SPDX-License-Identifier: MIT
> +#
> +# Variables that should be modified for your needs (could become args)
> +#
> +# api_url:
> +# the url for the Buildbot REST API on the AutoBuilder
> +# desired_branches:
> +# list of branches to find a matching build for
> +# desired_jobs:
> +# list of "Builders" on the AutoBuilder that perform the
> +# actual compatibility checks
> +# output_filename:
> +# name for the json file to output
> +#
> +# For "pretty" output:
> +# cat yp_compatible_layers.json | jq
> +
> +import sys
> +import os
> +
> +sys.path.insert(0,
> os.path.realpath(os.path.join(os.path.dirname(__file__), '..')))
> +
> +import argparse
> +import logging
> +import re
> +import requests
> +import json
> +
> +
> +def main():
> + # Setup logging
> + logging.basicConfig(level=logging.INFO, format="%(levelname)s:
> %(message)s")
> + logger = logging.getLogger('list-yocto-project-compatible-layers')
> +
> + parser = argparse.ArgumentParser(description='query Yocto AutoBuilder
> check-layer* jobs and create a json file of Yocto Compatible 2.0 layers')
> +
> + parser.add_argument("-n", "--num-builds",
> + help = "Limit the number of Auto Builder builds to query",
> + action="store", dest="num_builds", default=20)
> + parser.add_argument("-d", "--debug",
> + help = "Enable debug output",
> + action="store_const", const=logging.DEBUG, dest="loglevel",
> default=logging.INFO)
> + parser.add_argument("-q", "--quiet",
> + help = "Hide all output except error messages",
> + action="store_const", const=logging.ERROR, dest="loglevel")
> +
> + args = parser.parse_args()
> +
> + logger.setLevel(args.loglevel)
> +
> + api_url = 'https://autobuilder.yoctoproject.org/typhoon/api/v2'
> + desired_branches = {"dunfell", "kirkstone", "mickledore", "nanbield",
> "master"}
> + desired_jobs = {"check-layer", "check-layer-nightly"}
> + num_builds = args.num_builds
> + output_filename = 'yp_compatible_layers.json'
> + builder_id=dict()
> + # get info about a builder
> + # curl
> https://autobuilder.yoctoproject.org/typhoon/api/v2/builders?name=check-layer
> + builder_id['check_layer']=39
> + builder_id['check_layer_nightly']=121
> + # get the "api"
> + # curl
> https://autobuilder.yoctoproject.org/typhoon/api/v2/application.spec
> +
> + # get a specific step raw stdio log
> + # curl
> https://autobuilder.yoctoproject.org/typhoon/api/v2/builders/121/builds/1651/steps/11/logs/stdio/raw
> +
> + # filter for strings in the name of a step
> + # curl
> https://autobuilder.yoctoproject.org/typhoon/api/v2/builders/check-layer-nightly/builds/1651/steps?name__contains=Test
> + # curl
> https://autobuilder.yoctoproject.org/typhoon/api/v2/builders/check-layer-nightly/builds/1651/steps?name__contains=Run&field=name&field=number
> + #
> + # get the latest build (NOTE: - in front of 'number' does reverse
> order)
> + #
> https://autobuilder.yoctoproject.org/typhoon/api/v2/builders/check-layer-nightly/builds?order=-number&limit=1
> + #
> + # get list of properties for a build
> + #
> https://autobuilder.yoctoproject.org/typhoon/api/v2/builds/453477/property_list
> + #
> + # get the buildid of a build from builders/ so the builds/<buildid>
> api can be queried for property_list
> + # otherwise, we have no way to get the 'branch'
> + #
> https://autobuilder.yoctoproject.org/typhoon/api/v2/builders/check-layer-nightly/builds?limit=10&field=buildid
> +
> + def latest_builds(job, limit):
> + # limit to successful builds (however, this includes Skip)
> + url_str =
> f'{api_url}/builders/{job}/builds?state_string__contains=successful&limit={str(limit)}&order=-buildid&field=buildid&field=state_string'
> + r = requests.get(url_str)
> + return r
> +
> + # set objects are not JSON serializable, so we convert our set to a
> list
> + class YPCompatibleJSONEncoder(json.JSONEncoder):
> + def default(self, obj):
> + if isinstance(obj, set):
> + return list(obj)
> + return json.JSONEncoder.default(self, obj)
> +
> + #latest_20 = requests.get('
> https://autobuilder.yoctoproject.org/typhoon/api/v2/builders/check-layer-nightly/builds?limit=20&order=-buildid&field=buildid
> ')
> + compatible_dict = dict().fromkeys(desired_jobs,
> dict().fromkeys(desired_branches, set()))
> + for job in desired_jobs:
> + # we want a list we can decrement once a valid branch has been
> found
> + copy_desired_branches = desired_branches.copy()
> + logger.info(f'Processing at most {num_builds} builds for
> AutoBuilder job: {job}...')
> + for builds in latest_builds(job, num_builds).json()['builds']:
> + build_id=builds['buildid']
> + logger.debug(f'\tProcessing build: {build_id}...')
> + url_str =
> f'{api_url}/builds/{build_id}/property_list?name=yp_build_branch'
> + #r = requests.get('
> https://autobuilder.yoctoproject.org/typhoon/api/v2/builds/%s/property_list?name=yp_build_branch'
> % builds['buildid'])
> + r = requests.get(url_str)
> + try:
> + result_dict = r.json()['_properties'][0]
> + branch = result_dict['value'].replace('"','')
> + except:
> + pass
> + if branch in copy_desired_branches:
> + logger.info(f'\tProcessing branch: {branch}...')
> + url_str =
> f'{api_url}/builds/{build_id}/steps?name__contains=Run&field=name&field=number'
> + steps_req = requests.get(url_str)
> + steps_dict = steps_req.json()['steps']
> + for step in steps_dict:
> + try:
> + step_num = step['number']
> + # currently, meta-virtualization does not have
> 'cmds' so use a look ahead
> + repo_regex = re.compile(r"^Test (.+) YP
> Compatibility: Run(?: cmds)$")
> + repo = repo_regex.match(step['name']).group(1)
> + except:
> + pass
> + try:
> + url_str =
> f'{api_url}/builds/{build_id}/steps/{step_num}/logs/stdio/raw'
> + log_req = requests.get(url_str)
> + log = log_req.content.decode("utf-8")
> + # We only care about layers that have passed
> + layer_regex = re.compile(r"^INFO: (.+) ...
> PASS$", re.MULTILINE)
> + layers = layer_regex.findall(log)
> + if len(layers) > 0:
> + logger.info(f'\t\tDetected {layers}')
> + for layer in layers:
> + compatible_dict[job][branch].add(layer)
> + except:
> + sys.exit(1)
> + copy_desired_branches.remove(branch)
> + with open(output_filename, 'w') as f:
> + json.dump(compatible_dict, f, cls=YPCompatibleJSONEncoder)
> +
> + sys.exit(0)
> +
> +
> +if __name__ == "__main__":
> + main()
> --
> 2.34.1
>
>
>
>
>
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#61505): https://lists.yoctoproject.org/g/yocto/message/61505
Mute This Topic: https://lists.yoctoproject.org/mt/102231417/21656
Group Owner: yocto+ow...@lists.yoctoproject.org
Unsubscribe: https://lists.yoctoproject.org/g/yocto/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
