Hi,
Here is an updated patch, it fixes the issues you mentioned and adds
basic testing.
Raphael Hertzog wrote:
> I was thinking of having a generic task (i.e. a new class that we can
> inherit from in UpdateAutoRemovalsStatsTask) so that we
> don't duplicate the logic in __init__(), set_parameters, get_*_stats, etc
> and having a set of tests ready to run on any task built on top of the new
> generic task.
Yes we could have a GenericActionItemsTask.
Cheers,
Christophe
>From 1a98d4493b2e7a384fee6d18f9eb8d8784b7ff27 Mon Sep 17 00:00:00 2001
From: Christophe Siraut <d...@tobald.eu.org>
Date: Sun, 24 Aug 2014 13:38:46 +0200
Subject: [PATCH] vendor/debian: generate action items for auto-removals
---
.../templates/debian/autoremoval-action-item.html | 23 ++++++
distro_tracker/vendor/debian/tests.py | 72 ++++++++++++++++++
distro_tracker/vendor/debian/tracker_tasks.py | 85 ++++++++++++++++++++++
3 files changed, 180 insertions(+)
create mode 100644 distro_tracker/vendor/debian/templates/debian/autoremoval-action-item.html
diff --git a/distro_tracker/vendor/debian/templates/debian/autoremoval-action-item.html b/distro_tracker/vendor/debian/templates/debian/autoremoval-action-item.html
new file mode 100644
index 0000000..e0d8752
--- /dev/null
+++ b/distro_tracker/vendor/debian/templates/debian/autoremoval-action-item.html
@@ -0,0 +1,23 @@
+{% spaceless %}
+{% with bugs=item.extra_data.bugs %}
+{% with bugs_dependencies=item.extra_data.bugs_dependencies %}
+{% with buggy_dependencies=item.extra_data.buggy_dependencies %}
+
+<div>
+ <span>Version {{ item.extra_data.version }} of {{ item.package.name }} is marked for autoremoval from testing on {{ item.extra_data.removal_date }}. </span>
+
+ {% if bugs %}
+ <span>It is affected by {{ bugs|safe }}. </span>
+ {% endif %}
+
+ {% if bugs_dependencies %}
+ <span>It depends (transitively) on {{ buggy_dependencies|safe }}, affected by {{ bugs_dependencies|safe }}. </span>
+ {% endif %}
+
+ </span>You should try to prevent the removal by fixing these RC bugs.</span>
+</div>
+
+{% endwith %}
+{% endwith %}
+{% endwith %}
+{% endspaceless %}
diff --git a/distro_tracker/vendor/debian/tests.py b/distro_tracker/vendor/debian/tests.py
index 19e0586..646867e 100644
--- a/distro_tracker/vendor/debian/tests.py
+++ b/distro_tracker/vendor/debian/tests.py
@@ -62,6 +62,7 @@ from distro_tracker.vendor.debian.tracker_tasks import DebianWatchFileScannerUpd
from distro_tracker.vendor.debian.tracker_tasks import UpdateExcusesTask
from distro_tracker.vendor.debian.tracker_tasks import UpdateDebciStatusTask
from distro_tracker.vendor.debian.tracker_tasks import UpdateDebianDuckTask
+from distro_tracker.vendor.debian.tracker_tasks import UpdateAutoRemovalsStatsTask
from distro_tracker.vendor.debian.models import DebianContributor
from distro_tracker.vendor.debian.models import UbuntuPackage
from distro_tracker.vendor.debian.tracker_tasks import UpdateLintianStatsTask
@@ -4707,3 +4708,74 @@ class UpdateDebciStatusTaskTest(TestCase):
self.run_task()
self.assertEqual(ActionItem.objects.count(), 0)
+
+
+@mock.patch('distro_tracker.core.utils.http.requests')
+class UpdateAutoRemovalsStatsTaskTest(TestCase):
+ """
+ Tests for the :class:`distro_tracker.vendor.debian.tracker_tasks
+ .UpdateAutoRemovalsStatsTask` task.
+ """
+ def setUp(self):
+ self.dummy_package = SourcePackageName.objects.create(
+ name='dummy-package')
+ self.other_package = SourcePackageName.objects.create(
+ name='other-package')
+ self.autoremovals_data = """
+ dummy-package:
+ bugs:
+ - '12345'
+ removal_date: 2014-08-24 10:20:00
+ dummy-package2:
+ bugs:
+ - '123456'
+ removal_date: 2014-08-25 12:00:00
+ """
+
+ def run_task(self):
+ """
+ Runs the autoremovals status update task.
+ """
+ task = UpdateAutoRemovalsStatsTask()
+ task.execute()
+
+ def test_action_item_when_in_list(self, mock_requests):
+ """
+ Tests that an ActionItem is created for a package reported by
+ autoremovals.
+ """
+ set_mock_response(mock_requests, text=self.autoremovals_data)
+
+ self.run_task()
+ self.assertEqual(1, self.dummy_package.action_items.count())
+
+ def test_no_action_item_when_not_in_list(self, mock_requests):
+ """
+ Tests that no ActionItem is created for a package not reported by
+ autoremovals.
+ """
+ set_mock_response(mock_requests, text=self.autoremovals_data)
+
+ self.run_task()
+ self.assertEqual(0, self.other_package.action_items.count())
+
+ def test_action_item_is_dropped_when_autoremovals_reports_nothing_again(
+ self, mock_requests):
+ """
+ Tests that ActionItems are dropped when a package was previousy
+ reported but is now not reported anymore.
+ """
+ set_mock_response(mock_requests, text=self.autoremovals_data)
+ self.run_task()
+ self.assertEqual(1, self.dummy_package.action_items.count())
+
+ autoremovals_data = """
+ dummy-package3:
+ bugs:
+ - '1234567'
+ removal_date: 2014-08-22 12:21:00
+ """
+ set_mock_response(mock_requests, text=autoremovals_data)
+
+ self.run_task()
+ self.assertEqual(0, self.dummy_package.action_items.count())
diff --git a/distro_tracker/vendor/debian/tracker_tasks.py b/distro_tracker/vendor/debian/tracker_tasks.py
index 98b63d2..fc8de64 100644
--- a/distro_tracker/vendor/debian/tracker_tasks.py
+++ b/distro_tracker/vendor/debian/tracker_tasks.py
@@ -2122,3 +2122,88 @@ class UpdateDebciStatusTask(BaseTask):
# Remove action items for packages without failing tests.
ActionItem.objects.delete_obsolete_items(
[self.debci_action_item_type], packages)
+
+
+class UpdateAutoRemovalsStatsTask(BaseTask):
+ """
+ A task for updating autoremovals information on all packages.
+ """
+ ACTION_ITEM_TYPE_NAME = 'debian-autoremoval'
+ ACTION_ITEM_TEMPLATE = 'debian/autoremoval-action-item.html'
+ ITEM_DESCRIPTION = 'Marked for autoremoval on {removal_date}: {bugs}'
+
+ def __init__(self, force_update=False, *args, **kwargs):
+ super(UpdateAutoRemovalsStatsTask, self).__init__(*args, **kwargs)
+ self.force_update = force_update
+ self.action_item_type = ActionItemType.objects.create_or_update(
+ type_name=self.ACTION_ITEM_TYPE_NAME,
+ full_description_template=self.ACTION_ITEM_TEMPLATE)
+
+ def set_parameters(self, parameters):
+ if 'force_update' in parameters:
+ self.force_update = parameters['force_update']
+
+ def get_autoremovals_stats(self):
+ """
+ Retrieves and parses the autoremoval stats for all packages.
+ Autoremoval stats include the BTS bugs id.
+
+ :returns: A dict mapping package names to autoremoval stats.
+ """
+ content = get_resource_content(
+ 'https://udd.debian.org/cgi-bin/autoremovals.yaml.cgi')
+ if content:
+ return yaml.safe_load(six.BytesIO(content))
+
+ def update_action_item(self, package, stats):
+ """
+ Creates an :class:`ActionItem <distro_tracker.core.models.ActionItem>`
+ instance for the given type indicating that the package has an
+ autoremoval issue.
+ """
+ action_item = package.get_action_item_for_type(self.action_item_type)
+ if not action_item:
+ action_item = ActionItem(
+ package=package,
+ item_type=self.action_item_type,
+ severity=ActionItem.SEVERITY_HIGH)
+
+ bugs_dependencies = stats.get('bugs_dependencies', [])
+ buggy_dependencies = stats.get('buggy_dependencies', [])
+ all_bugs = stats['bugs'] + bugs_dependencies
+ link = '<a href="https://bugs.debian.org/{}";>{}</a>'
+
+ action_item.short_description = self.ITEM_DESCRIPTION.format(
+ removal_date=stats['removal_date'].strftime('%d %B'),
+ bugs=', '.join(link.format(bug, bug) for bug in all_bugs))
+
+ action_item.extra_data = {
+ 'stats': stats,
+ 'removal_date': stats['removal_date'].strftime('%a %d %b %Y'),
+ 'bugs': ', '.join(link.format(bug, bug) for bug in stats['bugs']),
+ 'bugs_dependencies': ', '.join(
+ link.format(bug, bug) for bug in bugs_dependencies),
+ 'buggy_dependencies': ' and '.join(
+ ['<a href="/pkg/{}">{}</a>'.format(
+ reverse(
+ 'dtracker-package-page',
+ kwargs={'package_name': p}),
+ p) for p in buggy_dependencies])}
+ action_item.save()
+
+ def execute(self):
+ autoremovals_stats = self.get_autoremovals_stats()
+ if autoremovals_stats is None:
+ # Nothing to do: cached content up to date
+ return
+
+ ActionItem.objects.delete_obsolete_items(
+ item_types=[self.action_item_type],
+ non_obsolete_packages=autoremovals_stats.keys())
+
+ packages = SourcePackageName.objects.filter(
+ name__in=autoremovals_stats.keys())
+ packages = packages.prefetch_related('action_items')
+
+ for package in packages:
+ self.update_action_item(package, autoremovals_stats[package.name])
--
2.1.0.rc1
