Hi, I wrote a quick and dirty Python 3 script to track upstream package version including last GIT commit and commit date, so you can compare Hedora's version with upstream. This is slightly better than Anitya as it supports commit.
It takes as input a file containing one Fedora package per line and as output a "TODO Golang update.csv". I have not put this on Github yet but if you're interested in improving it, I can do that. I'm not a programming specialist so I'm sure there is tons of errors and dirty hack in there. I'm joining the script and an example output from almost all packages we have. Best regards, Robert-André
#!/usr/bin/python3
import asyncio
import aiohttp
import asyncio_throttle
import backoff
from bs4 import BeautifulSoup
import time
import re
import sys
def all_go_packages(filename):
with open(filename) as f:
pkgs = [line.rstrip() for line in f if line != ""]
return pkgs
def write_to_file(data):
f = open("TODO Golang update.csv", "a")
f.write(data)
f.close()
def erase():
f = open("TODO Golang update.csv", "w")
f.write("")
f.close()
def extract_commit_info(fedrel):
reldata = re.search("(\d{8})git([a-f0-9]+)", fedrel, flags=re.MULTILINE)
if reldata:
return reldata.group(1), reldata.group(2)
else:
return "", ""
def convert_date(datein):
try:
dateout = time.strftime("%Y%m%d", time.strptime(datein, "%b %d, %Y"))
except:
return datein
return dateout
def convert_version(version_string):
version_data = re.search(
"v([\d+][\.\d]+)(\-\d{14}\-)*([a-f0-9]*)",
version_string,
flags=re.MULTILINE,
)
if version_data:
version = version_data.group(1)
else:
return "0", ""
if "0.0.0" in version:
version = 0
if version_data.group(3):
commit = version_data.group(3)
else:
commit = ""
return str(version), commit
@backoff.on_exception(
backoff.expo, (aiohttp.ClientError, asyncio.TimeoutError), max_time=30
)
async def get_fedora_version(pkg):
if pkg:
url = f"https://apps.fedoraproject.org/mdapi/koji/srcpkg/{pkg}";
timeout = aiohttp.ClientTimeout(total=60)
async with aiohttp.ClientSession(timeout=timeout) as session:
try:
async with session.get(url) as resp:
if resp.status == 404:
return "unknown", "unknown"
resp.raise_for_status()
data = await resp.json()
ver = data["version"]
rel = data["release"]
return str(ver), str(rel)
except asyncio.TimeoutError:
print(f"Timeout get_fedora_version {pkg}")
except:
raise
@backoff.on_exception(
backoff.expo, (aiohttp.ClientError, asyncio.TimeoutError), max_time=30
)
async def get_fedora_goipath(pkg):
if pkg:
url = f"https://src.fedoraproject.org/rpms/{pkg}/raw/master/f/{pkg}.spec";
timeout = aiohttp.ClientTimeout(total=60)
async with aiohttp.ClientSession(timeout=timeout) as session:
try:
async with session.get(url) as resp:
if resp.status == 404:
return "unknown"
resp.raise_for_status()
spec = await resp.text()
ipath = re.search("(goipath)\s+(.*)$", spec, flags=re.MULTILINE)
if ipath:
return ipath.group(2)
else:
return "unknown"
except asyncio.TimeoutError:
print(f"Timeout get_fedora_goipath {pkg}")
except:
raise
@backoff.on_exception(
backoff.expo, (aiohttp.ClientError, asyncio.TimeoutError), max_time=30
)
async def get_upstream_version(goipath):
url = f"https://pkg.go.dev/mod/{goipath}?tab=versions";
timeout = aiohttp.ClientTimeout(total=60)
async with aiohttp.ClientSession(timeout=timeout) as session:
try:
async with session.get(url) as resp:
if resp.status == 404 or resp.status == 400:
return 0, "unknown", "unknown"
resp.raise_for_status()
page = await resp.text()
soup = BeautifulSoup(page, "html.parser")
li_elmt = soup.findAll("li", {"class": "Versions-item"})[0]
version, commit = convert_version(li_elmt.a.string)
commitdate = convert_date(li_elmt.span.string[3:])
return version, commit, commitdate
except asyncio.TimeoutError:
print(f"Timeout get_upstream_version {pkg}")
return get_upstream_version(goipath)
except:
raise
async def check_version(pkg):
fedora_version, fedora_release = await get_fedora_version(pkg)
goipath = await get_fedora_goipath(pkg)
fedora_date, fedora_commit = extract_commit_info(fedora_release)
if goipath and not "unknown" in goipath:
upstream_version, upstream_commit, upstream_date = await get_upstream_version(
goipath
)
else:
write_to_file(
f"{pkg} Fedora,{fedora_version},{fedora_date},{fedora_commit}\nunknown Upstream,unknown,unknown,unknown\n\n"
)
return
if fedora_version == "0" and upstream_version == "0":
if not fedora_commit in upstream_commit:
write_to_file(
f"{pkg} Fedora,{fedora_version},{fedora_date},{fedora_commit}\n{goipath} Upstream,{upstream_version},{upstream_date},{upstream_commit}\n\n"
)
return
elif fedora_version != upstream_version:
if not fedora_commit in upstream_commit:
write_to_file(
f"{pkg} Fedora,{fedora_version},{fedora_date},{fedora_commit}\n{goipath} Upstream,{upstream_version},{upstream_date},{upstream_commit}\n\n"
)
return
return
async def main(loop):
erase()
no_concurrent = 5
tasks = set()
pkgs = all_go_packages(sys.argv[1])
i = 0
while i < len(pkgs):
if len(tasks) >= no_concurrent:
# Wait for some download to finish before adding a new one
_done, tasks = await asyncio.wait(
tasks, return_when=asyncio.FIRST_COMPLETED
)
tasks.add(loop.create_task(check_version(pkgs[i])))
i += 1
# Wait for the remaining downloads to finish
await asyncio.wait(tasks)
if __name__ == "__main__":
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(main(loop))
finally:
loop.close()
<<attachment: TODO_Golang_update.csv.zip>>
_______________________________________________ golang mailing list -- golang@lists.fedoraproject.org To unsubscribe send an email to golang-le...@lists.fedoraproject.org Fedora Code of Conduct: https://docs.fedoraproject.org/en-US/project/code-of-conduct/ List Guidelines: https://fedoraproject.org/wiki/Mailing_list_guidelines List Archives: https://lists.fedoraproject.org/archives/list/golang@lists.fedoraproject.org
