This patch adds asynchronous task execution using a Celery backend
and RabbitMQ task queue.
It updates the README to reflect the installation and configuration
of a basic RabbitMQ setup, adds a 'tasks.py' file to contain task
definitions, updates the 'edit_layer_view' function to send
emails to administrators about new and updated layers asynchronously,
and modifies the 'settings.py' to include a default configuration
for a RabbitMQ connection.
---
README | 10 +++++++++-
TODO | 1 -
layerindex/tasks.py | 24 ++++++++++++++++++++++++
layerindex/views.py | 7 +++----
requirements.txt | 1 +
settings.py | 4 ++++
6 files changed, 41 insertions(+), 6 deletions(-)
create mode 100644 layerindex/tasks.py
diff --git a/README b/README
index 62f739d..c7f7409 100644
--- a/README
+++ b/README
@@ -14,6 +14,7 @@ In order to make use of this application you will need:
* Python 3.4+
* Django 1.8.x - tested with 1.8.17; newer versions may work, but
the application has not been tested with 1.9 or newer.
+* RabbitMQ 3.6.x - tested with 3.6.10.
* For production usage, a web server set up to host Django applications
(not needed for local-only testing)
* A database supported by Django (SQLite, MySQL, etc.). Django takes
@@ -41,7 +42,9 @@ Setup instructions:
1. Edit settings.py to specify a database, EMAIL_HOST, SECRET_KEY and
other settings specific to your installation. Ensure you set
LAYER_FETCH_DIR to an absolute path to a location with sufficient
- space for fetching layer repositories.
+ space for fetching layer repositories. Modify RABBIT_BROKER
+ and RABBIT_BACKEND to reflect the settings used by your RabbitMQ
+ server.
2. Run the following commands within the layerindex-web directory to
initialise the database:
@@ -64,6 +67,11 @@ Setup instructions:
production you need to use a proper web server and have DEBUG set
to False.
+ 3.1. In order to process asynchronous tasks like sending email,
+ you will need to run a Celery worker:
+
+ celery -A layerindex.tasks worker --loglevel=info
+
4. You'll need to add at least the openembedded-core layer to the
database, or some equivalent that contains conf/bitbake.conf for
the base system configuration. To add this, follow these steps:
diff --git a/TODO b/TODO
index 186219f..29986ac 100644
--- a/TODO
+++ b/TODO
@@ -27,7 +27,6 @@ Other
* Show layer type in layer detail?
* Usage links in list page?
* Subdirs in list page?
-* Prevent SMTP failures from breaking submission process
* Query backend service i.e. special URL to query information for external
apps/scripts
* Add comparison to duplicates page
* Create simple script to check for unlisted layer subdirectories in all repos
diff --git a/layerindex/tasks.py b/layerindex/tasks.py
new file mode 100644
index 0000000..de80804
--- /dev/null
+++ b/layerindex/tasks.py
@@ -0,0 +1,24 @@
+from celery import Celery
+from django.core.mail import EmailMessage
+from . import utils
+import os
+import time
+
+try:
+ import settings
+except ImportError:
+ # not in a full django env, so settings is inaccessible.
+ # setup django to access settings.
+ utils.setup_django()
+ import settings
+
+tasks = Celery('layerindex',
+ broker=settings.RABBIT_BROKER,
+ backend=settings.RABBIT_BACKEND)
+
+@tasks.task
+def send_email(subject, text_content, from_email=settings.DEFAULT_FROM_EMAIL,
to_emails=[]):
+ # We seem to need to run this within the task
+ utils.setup_django()
+ msg = EmailMessage(subject, text_content, from_email, to_emails)
+ msg.send()
diff --git a/layerindex/views.py b/layerindex/views.py
index 1661cb3..8eac866 100644
--- a/layerindex/views.py
+++ b/layerindex/views.py
@@ -19,7 +19,6 @@ from layerindex.forms import EditLayerForm,
LayerMaintainerFormSet, EditNoteForm
from django.db import transaction
from django.contrib.auth.models import User, Permission
from django.db.models import Q, Count, Sum
-from django.core.mail import EmailMessage
from django.template.loader import get_template
from django.template import Context
from django.utils.decorators import method_decorator
@@ -28,6 +27,7 @@ from django.contrib import messages
from reversion.models import Revision
from . import utils
from . import simplesearch
+from . import tasks
import settings
from django.dispatch import receiver
import reversion
@@ -163,7 +163,7 @@ def edit_layer_view(request, template_name,
branch='master', slug=None):
# Send email
plaintext = get_template('layerindex/submitemail.txt')
perm = Permission.objects.get(codename='publish_layer')
- users = User.objects.filter(Q(groups__permissions=perm) |
Q(user_permissions=perm) ).distinct()
+ users = User.objects.filter(Q(groups__permissions=perm) |
Q(user_permissions=perm) | Q(is_superuser=True)).distinct()
for user in users:
if user.first_name:
user_name = user.first_name
@@ -181,8 +181,7 @@ def edit_layer_view(request, template_name,
branch='master', slug=None):
from_email = settings.SUBMIT_EMAIL_FROM
to_email = user.email
text_content = plaintext.render(d)
- msg = EmailMessage(subject, text_content, from_email,
[to_email])
- msg.send()
+ tasks.send_email.apply_async((subject, text_content,
from_email, [to_email]))
return HttpResponseRedirect(reverse('submit_layer_thanks'))
messages.success(request, 'Layer %s saved successfully.' %
layeritem.name)
if return_url:
diff --git a/requirements.txt b/requirements.txt
index f80e5d4..7cd741b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,4 @@
+celery==3.1.25
Django==1.8.17
django-cors-headers==1.1.0
django-nvd3==0.9.7
diff --git a/settings.py b/settings.py
index 92146f8..e26f4b2 100644
--- a/settings.py
+++ b/settings.py
@@ -224,3 +224,7 @@ FORCE_REVIEW_HTTPS = False
# Settings for layer submission feature
SUBMIT_EMAIL_FROM = 'nore...@example.com'
SUBMIT_EMAIL_SUBJECT = 'OE Layerindex layer submission'
+
+# RabbitMQ settings
+RABBIT_BROKER = 'amqp://'
+RABBIT_BACKEND = 'rpc://'
--
2.7.4
--
_______________________________________________
yocto mailing list
yocto@yoctoproject.org
https://lists.yoctoproject.org/listinfo/yocto
