Hello,
Here is a patch adding support for Novell, KDE (for KOffice/Calligra), GNOME
(for Gnumeric), AbiSource
(for AbiWord) bugzillas and Launchpad to
bin/get-bugzilla-attachments-by-mimetype script as required by easyhack#46538.
However, there two glitches and workarounds:
1- As stated in the code, Novell bugzilla requires users to log in to bugzilla
in
order to get full information about the bugs such as attachment bodies etc.
Since
Novell bugzilla uses some proprietary login extensions it's hard to implement
log in
mechanism and use it via xmlrpc API. python-bugzilla claims that it supports
Novell
bugzilla login but I could not make it work. As a workaround, comments
containing
"Created an attachment (id=xxxxxxx)" pattern are parsed and attachments are
downloaded
using urllib. Attachments are filtered using mimetype info obtained from http
headers.
2- For launchpad, launchpadlib python module is used. But currently launchpad
api
does not support really advanced search like bugzilla. For example, it's not
possible
to search for bugs having application/pdf attachments. So, again a workaround
is used. Code
iterates over all the bugs of the *libreoffice* source package and check their
attachments.
---
bin/get-bugzilla-attachments-by-mimetype | 278 ++++++++++++++++++++----------
1 file changed, 191 insertions(+), 87 deletions(-)
diff --git a/bin/get-bugzilla-attachments-by-mimetype
b/bin/get-bugzilla-attachments-by-mimetype
index 0b18060..7316f0a 100755
--- a/bin/get-bugzilla-attachments-by-mimetype
+++ b/bin/get-bugzilla-attachments-by-mimetype
@@ -39,6 +39,7 @@
import urllib
import feedparser
import base64
+import re
import os, os.path
import xmlrpclib
from xml.dom import minidom
@@ -83,6 +84,42 @@ def get_from_bug_url_via_xml(url, mimetype, prefix, suffix):
break
attachmentid += 1
+def get_novell_bug_via_xml(url, mimetype, prefix, suffix):
+ id = url.rsplit('=', 2)[1]
+ print "id is", prefix, id, suffix
+ if os.path.isfile(suffix + '/' + prefix + id + '-1.' + suffix):
+ print "assuming", id, "is up to date"
+ else:
+ print "parsing", id
+ sock = urlopen_retry(url+"&ctype=xml")
+ dom = minidom.parse(sock)
+ sock.close()
+ attachmentid=1
+ for comment in dom.getElementsByTagName('thetext'):
+ commentText = comment.firstChild.nodeValue
+ match = re.search(r".*Created an attachment \(id=([0-9]+)\)",
commentText)
+ if not match:
+ continue
+
+ realAttachmentId = match.group(1)
+ handle = urlopen_retry(novellattach + realAttachmentId)
+ if not handle:
+ print "attachment %s is not accessible", realAttachmentId
+ continue
+
+ remoteMime = handle.info().gettype()
+ if remoteMime != mimetype:
+ print "skipping"
+ continue
+
+ download = suffix + '/' + prefix + id + '-' + str(attachmentid) +
'.' + suffix
+ print 'downloading as', download
+ f = open(download, 'w')
+ f.write(handle.read())
+ f.close()
+ break
+ attachmentid += 1
+
def get_through_rpc_query(rpcurl, showurl, mimetype, prefix, suffix):
try:
proxy = xmlrpclib.ServerProxy(rpcurl)
@@ -109,108 +146,164 @@ def get_through_rss_query_url(url, mimetype, prefix,
suffix):
except:
pass
d = feedparser.parse(url)
+
+ #Getting detailed bug information and downloading an attachment body is
not possible without logging in to Novell bugzilla
+ #get_novell_bug_via_xml function is a workaround for that situation
+ get_bug_function = get_novell_bug_via_xml if prefix == "novell" else
get_from_bug_url_via_xml
+
for entry in d['entries']:
- get_from_bug_url_via_xml(entry['id'], mimetype, prefix, suffix)
+ get_bug_function(entry['id'], mimetype, prefix, suffix)
def get_through_rss_query(queryurl, mimetype, prefix, suffix):
url = queryurl +
'?query_format=advanced&field0-0-0=attachments.mimetype&type0-0-0=equals&value0-0-0='
+ escape(mimetype) + '&ctype=rss'
print 'url is', url
get_through_rss_query_url(url, mimetype, prefix, suffix)
+def get_launchpad_bugs(prefix):
+ #launchpadlib python module is required to download launchpad attachments
+ from launchpadlib.launchpad import Launchpad
+
+ launchpad = Launchpad.login_anonymously("attachmentdownload", "production")
+ ubuntu = launchpad.distributions["ubuntu"]
+
+ #since searching bugs having attachments with specific mimetypes is not
available in launchpad API
+ #we're iterating over all bugs of the libreoffice source package
+ libo = ubuntu.getSourcePackage(name="libreoffice")
+ libobugs = libo.getBugTasks()
+
+ for bugtask in libobugs:
+ bug = bugtask.bug
+ id = str(bug.id)
+ print "parsing ", id, "status:", bugtask.status, "title:",
bug.title[:50]
+ attachmentid = 1
+ for attachment in bug.attachments:
+ handle = attachment.data.open()
+ if not handle.content_type in mimetypes:
+ #print "skipping"
+ continue
+
+ suffix = mimetypes[handle.content_type]
+ if not os.path.isdir(suffix):
+ try:
+ os.mkdir(suffix)
+ except:
+ pass
+
+ download = suffix + '/' + prefix + id + '-' + str(attachmentid) +
'.' + suffix
+
+ if os.path.isfile(download):
+ print "assuming", id, "is up to date"
+ break
+
+ print 'mimetype is', handle.content_type, 'downloading as',
download
+
+ f = open(download, "w")
+ f.write(handle.read())
+ f.close()
+ attachmentid += 1
freedesktop = 'http://bugs.freedesktop.org/buglist.cgi'
+abisource = 'http://bugzilla.abisource.com/buglist.cgi' #added for abiword
+gnome = 'http://bugzilla.gnome.org/buglist.cgi' # added for gnumeric
+kde = 'http://bugs.kde.org/buglist.cgi' # added for koffice/calligra
openoffice = 'https://issues.apache.org/ooo/buglist.cgi'
redhatrpc = 'https://bugzilla.redhat.com/xmlrpc.cgi'
redhatbug = 'https://bugzilla.redhat.com/show_bug.cgi?id='
-novell = 'https://bugzilla.novell.com/buglist.cgi'
mozilla = 'https://bugzilla.mozilla.org/buglist.cgi'
-mimetypes = [
+#Novell Bugzilla requires users to log in in order to get details of the bugs
such as attachment bodies etc.
+#As a dirty workaround, we parse comments containing "Created an attachment
(id=xxxxxx)" and download attachments manually
+#python-bugzilla claims that it supports Novell bugzilla login but it's not
working right now and novell bugzilla login
+#system is a nightmare
+novellattach = 'https://bugzilla.novell.com/attachment.cgi?id='
+novell = 'https://bugzilla.novell.com/buglist.cgi'
+
+mimetypes = {
# ODF
- ('application/vnd.oasis.opendocument.base', 'odb'),
- ('application/vnd.oasis.opendocument.database', 'odb'),
- ('application/vnd.oasis.opendocument.chart', 'odc'),
- ('application/vnd.oasis.opendocument.chart-template', 'otc'),
- ('application/vnd.oasis.opendocument.formula', 'odf'),
- ('application/vnd.oasis.opendocument.formula-template', 'otf'),
- ('application/vnd.oasis.opendocument.graphics', 'odg'),
- ('application/vnd.oasis.opendocument.graphics-template', 'otg'),
- ('application/vnd.oasis.opendocument.graphics-flat-xml', 'fodg'),
- ('application/vnd.oasis.opendocument.presentation', 'odp'),
- ('application/vnd.oasis.opendocument.presentation-template', 'otp'),
- ('application/vnd.oasis.opendocument.presentation-flat-xml', 'fodp'),
- ('application/vnd.oasis.opendocument.spreadsheet', 'ods'),
- ('application/vnd.oasis.opendocument.spreadsheet-template', 'ots'),
- ('application/vnd.oasis.opendocument.spreadsheet-flat-xml', 'fods'),
- ('application/vnd.oasis.opendocument.text', 'odt'),
- ('application/vnd.oasis.opendocument.text-flat-xml', 'fodt'),
- ('application/vnd.oasis.opendocument.text-master', 'odm'),
- ('application/vnd.oasis.opendocument.text-template', 'ott'),
- ('application/vnd.oasis.opendocument.text-web', 'oth'),
+ 'application/vnd.oasis.opendocument.base': 'odb',
+ 'application/vnd.oasis.opendocument.database': 'odb',
+ 'application/vnd.oasis.opendocument.chart': 'odc',
+ 'application/vnd.oasis.opendocument.chart-template': 'otc',
+ 'application/vnd.oasis.opendocument.formula': 'odf',
+ 'application/vnd.oasis.opendocument.formula-template': 'otf',
+ 'application/vnd.oasis.opendocument.graphics': 'odg',
+ 'application/vnd.oasis.opendocument.graphics-template': 'otg',
+ 'application/vnd.oasis.opendocument.graphics-flat-xml': 'fodg',
+ 'application/vnd.oasis.opendocument.presentation': 'odp',
+ 'application/vnd.oasis.opendocument.presentation-template': 'otp',
+ 'application/vnd.oasis.opendocument.presentation-flat-xml': 'fodp',
+ 'application/vnd.oasis.opendocument.spreadsheet': 'ods',
+ 'application/vnd.oasis.opendocument.spreadsheet-template': 'ots',
+ 'application/vnd.oasis.opendocument.spreadsheet-flat-xml': 'fods',
+ 'application/vnd.oasis.opendocument.text': 'odt',
+ 'application/vnd.oasis.opendocument.text-flat-xml': 'fodt',
+ 'application/vnd.oasis.opendocument.text-master': 'odm',
+ 'application/vnd.oasis.opendocument.text-template': 'ott',
+ 'application/vnd.oasis.opendocument.text-web': 'oth',
# OOo XML
- ('application/vnd.sun.xml.base', 'odb'),
- ('application/vnd.sun.xml.calc', 'sxc'),
- ('application/vnd.sun.xml.calc.template', 'stc'),
- ('application/vnd.sun.xml.chart', 'sxs'),
- ('application/vnd.sun.xml.draw', 'sxd'),
- ('application/vnd.sun.xml.draw.template', 'std'),
- ('application/vnd.sun.xml.impress', 'sxi'),
- ('application/vnd.sun.xml.impress.template', 'sti'),
- ('application/vnd.sun.xml.math', 'sxm'),
- ('application/vnd.sun.xml.writer', 'sxw'),
- ('application/vnd.sun.xml.writer.global', 'sxg'),
- ('application/vnd.sun.xml.writer.template', 'stw'),
- ('application/vnd.sun.xml.writer.web', 'stw'),
+ 'application/vnd.sun.xml.base': 'odb',
+ 'application/vnd.sun.xml.calc': 'sxc',
+ 'application/vnd.sun.xml.calc.template': 'stc',
+ 'application/vnd.sun.xml.chart': 'sxs',
+ 'application/vnd.sun.xml.draw': 'sxd',
+ 'application/vnd.sun.xml.draw.template': 'std',
+ 'application/vnd.sun.xml.impress': 'sxi',
+ 'application/vnd.sun.xml.impress.template': 'sti',
+ 'application/vnd.sun.xml.math': 'sxm',
+ 'application/vnd.sun.xml.writer': 'sxw',
+ 'application/vnd.sun.xml.writer.global': 'sxg',
+ 'application/vnd.sun.xml.writer.template': 'stw',
+ 'application/vnd.sun.xml.writer.web': 'stw',
# MSO
- ('application/rtf', 'rtf'),
- ('text/rtf', 'rtf'),
- ('application/msword', 'doc'),
- ('application/vnd.ms-powerpoint', 'ppt'),
- ('application/vnd.ms-excel', 'xls'),
- ('application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
'xlsx'),
- ('application/vnd.openxmlformats-officedocument.spreadsheetml.template',
'xltx'),
-
('application/vnd.openxmlformats-officedocument.presentationml.presentation',
'pptx'),
- ('application/vnd.openxmlformats-officedocument.presentationml.template',
'ppotx'),
- ('application/vnd.openxmlformats-officedocument.presentationml.slideshow',
'ppsx'),
-
('application/vnd.openxmlformats-officedocument.wordprocessingml.document',
'docx'),
-
('application/vnd.openxmlformats-officedocument.wordprocessingml.template',
'dotx'),
- ('application/vnd.visio', 'vsd'),
+ 'application/rtf': 'rtf',
+ 'text/rtf': 'rtf',
+ 'application/msword': 'doc',
+ 'application/vnd.ms-powerpoint': 'ppt',
+ 'application/vnd.ms-excel': 'xls',
+ 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet':
'xlsx',
+ 'application/vnd.openxmlformats-officedocument.spreadsheetml.template':
'xltx',
+
'application/vnd.openxmlformats-officedocument.presentationml.presentation':
'pptx',
+ 'application/vnd.openxmlformats-officedocument.presentationml.template':
'ppotx',
+ 'application/vnd.openxmlformats-officedocument.presentationml.slideshow':
'ppsx',
+ 'application/vnd.openxmlformats-officedocument.wordprocessingml.document':
'docx',
+ 'application/vnd.openxmlformats-officedocument.wordprocessingml.template':
'dotx',
+ 'application/vnd.visio': 'vsd',
# W3C
- ('application/xhtml+xml', 'xhtml'),
- ('application/mathml+xml', 'mml'),
- ('text/html', 'html'),
- ('application/docbook+xml', 'docbook'),
+ 'application/xhtml+xml': 'xhtml',
+ 'application/mathml+xml': 'mml',
+ 'text/html': 'html',
+ 'application/docbook+xml': 'docbook',
# misc
- ('text/spreadsheet', 'slk'),
- ('application/pdf', 'pdf'),
- ('application/vnd.corel-draw', 'cdr'),
- ('application/vnd.lotus-wordpro', 'lwp'),
- ('application/vnd.lotus-1-2-3', 'wks'),
- ('application/vnd.wordperfect', 'wpd'),
- ('application/vnd.ms-works', 'wps'),
- ('application/x-hwp', 'hwp'),
- ('application/x-aportisdoc', 'pdb'),
- ('application/x-pocket-word', 'psw'),
- ('application/x-t602', '602'),
+ 'text/spreadsheet': 'slk',
+ 'application/pdf': 'pdf',
+ 'application/vnd.corel-draw': 'cdr',
+ 'application/vnd.lotus-wordpro': 'lwp',
+ 'application/vnd.lotus-1-2-3': 'wks',
+ 'application/vnd.wordperfect': 'wpd',
+ 'application/vnd.ms-works': 'wps',
+ 'application/x-hwp': 'hwp',
+ 'application/x-aportisdoc': 'pdb',
+ 'application/x-pocket-word': 'psw',
+ 'application/x-t602': '602',
# binfilter
- ('application/x-starcalc', 'sdc'),
- ('application/vnd.stardivision.calc', 'sdc5'),
- ('application/x-starchart', 'sds'),
- ('application/vnd.stardivision.chart', 'sds5'),
- ('application/x-stardraw', 'sdd_d'),
- ('application/vnd.stardivision.draw', 'sda5'),
- ('application/x-starimpress', 'sdd_i'),
- ('application/vnd.stardivision.impress', 'sdd5'),
- ('application/vnd.stardivision.impress-packed', 'sdp5'),
- ('application/x-starmath', 'smf'),
- ('application/vnd.stardivision.math', 'smf5'),
- ('application/x-starwriter', 'sdw'),
- ('application/vnd.stardivision.writer', 'sdw5'),
- ('application/vnd.stardivision.writer-global', 'sgl5'),
+ 'application/x-starcalc': 'sdc',
+ 'application/vnd.stardivision.calc': 'sdc5',
+ 'application/x-starchart': 'sds',
+ 'application/vnd.stardivision.chart': 'sds5',
+ 'application/x-stardraw': 'sdd_d',
+ 'application/vnd.stardivision.draw': 'sda5',
+ 'application/x-starimpress': 'sdd_i',
+ 'application/vnd.stardivision.impress': 'sdd5',
+ 'application/vnd.stardivision.impress-packed': 'sdp5',
+ 'application/x-starmath': 'smf',
+ 'application/vnd.stardivision.math': 'smf5',
+ 'application/x-starwriter': 'sdw',
+ 'application/vnd.stardivision.writer': 'sdw5',
+ 'application/vnd.stardivision.writer-global': 'sgl5',
# unusual image mimetypes
- ('image/cgm', 'cgm'),
- ('image/x-targa', 'tga'),
-]
+ 'image/cgm': 'cgm',
+ 'image/x-targa': 'tga',
+}
# disabled for now, this would download gigs of pngs/jpegs...
common_image_mimetypes = [
@@ -241,16 +334,27 @@ common_image_mimetypes = [
('image/png', 'png'),
]
-for (mimetype,extension) in mimetypes:
+for (mimetype,extension) in mimetypes.items():
+ get_through_rss_query(novell, mimetype, "novell", extension)
+
+for (mimetype,extension) in mimetypes.items():
+ get_through_rss_query(kde, mimetype, "kde", extension)
+
+for (mimetype,extension) in mimetypes.items():
+ get_through_rss_query(gnome, mimetype, "gnome", extension)
+
+for (mimetype,extension) in mimetypes.items():
+ get_through_rss_query(abisource, mimetype, "abi", extension)
+
+for (mimetype,extension) in mimetypes.items():
get_through_rss_query(freedesktop, mimetype, "fdo", extension)
-for (mimetype,extension) in mimetypes:
+for (mimetype,extension) in mimetypes.items():
get_through_rpc_query(redhatrpc, redhatbug, mimetype, "rhbz", extension)
-#to-do, get attachments some other way, not inline in xml
-#get_through_rss_query(novell, 'application/msword', "n", "doc")
-
-for (mimetype,extension) in mimetypes:
+for (mimetype,extension) in mimetypes.items():
get_through_rss_query(openoffice, mimetype, "ooo", extension)
+get_launchpad_bugs("lp")
+
# vim:set shiftwidth=4 softtabstop=4 expandtab:
--
1.7.10
_______________________________________________
LibreOffice mailing list
LibreOffice@lists.freedesktop.org
http://lists.freedesktop.org/mailman/listinfo/libreoffice
