-----BEGIN PGP SIGNED MESSAGE----- Hash: SHA1 Attached is a working solution to this bug. Right now parsing the JSON Security Tracker Information results in a one definition per CVE. I hope to reduce this to one definition per package. (Shouldn't be too much work) Also included in the tar is are more updates to the legacy versions of parsing the dsa/wml files. If you need a debian oval interpreter I previously attached one to this forum.
I encourage you to use -h option for the JSON parsing script as downloading the security tracker each time is cumbersome. shasums: 5d13f72cccc6ad1774e2437c98c26bebfc2100ac 68910 diff.txt 57fa67e77ed5212d65deca0483b7ac90b5bb7203 21423 oval.tar.gz - -- Nicholas Luedtke Linux for HPE Helion OpenStack, Hewlett Packard Enterprise -----BEGIN PGP SIGNATURE----- Version: GnuPG v2.0.22 (GNU/Linux) iQIcBAEBAgAGBQJWRkcEAAoJEL+bDtw6splIE7AP/AwvpgvxPRV6ULYtOzJ8CWfH K2Wqs8jdfmLffl+SqSML1Mp8N/SrUKnqOxPuD+OgYjB2WsiBTz+5tU6NOw7Ytos0 bzuswxTQ1ybIzK/VILv5+ADRmeW0kjN7abX/5XF2ozdxsOdMb1Xjie9eEF2elYuM NWIJSfY/4WTwMQKz+8bfL22Z11W/SmNybhKBw+xVnjUd2R4fLSHKxJuAy2QB5sJG Fqov4f5nyVN4Vmcl9KjFr035YwphfqVeOHP2dJ6/BKWfMy6TxWe6I1nw90f9F1NL XIYNJ92e3ekR6P3zY4i7JQTLCrHbUKSDYzslpIUy+QWUsS/CMbVROOE4ETwY2Ze+ 3ZDQnoDN/aaXD9YOpEwp2hWYkCWB1KWBV26OzPhiybaW/Ev9af70SllGFNAPScQC qRezY8gQpRpB1rtDNpsdydQG3Le4AN+qPcw5QvVgrxyTr8Q9ds0OvNXjMl0Ya4za kxVRUx/iRFBj42n9yv9dlQb+mLIGg+6vGGVdwbGVizVdknBg/AS/QjWeXhe3rrpf JoPo9B2ljl9BLq8fKKM2RGJjOMjR76BO/ioqZRM/0r6uWLtgbAxsDfb1xm6f7dCA mFX+RbFYRIwL70f6hfPG+RqSBG0tAQidI/xHRfDmnvjVmd6RRy48P+zz/5chSBKu 9NPAkeFETaxy6xR8lSCZ =rDgk -----END PGP SIGNATURE-----
oval.tar.gz
Description: oval.tar.gz
Binary files oval_old/oval/definition/differ.pyc and
oval/oval/definition/differ.pyc differ
diff -urN oval_old/oval/definition/generator.py
oval/oval/definition/generator.py
--- oval_old/oval/definition/generator.py 2011-10-12 15:14:34.000000000
-0600
+++ oval/oval/definition/generator.py 2015-11-13 11:20:30.043834000 -0700
@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
-# oval.definitio.generator - generate well-formed xml file with
+# oval.definition.generator - generate well-formed xml file with
# OVAL definitions of Debian Security Advisories.
# Use various optimizations to minimize result XML
#
@@ -19,44 +19,44 @@
RE_XML_ILLEGAL = u'([\u0000-\u0008\u000b-\u000c\u000e-\u001f\ufffe-\uffff])' +
u'|' + u'([%s-%s][^%s-%s])|([^%s-%s][%s-%s])|([%s-%s]$)|(^[%s-%s])' %
(unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff),
unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff),
unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff))
regex = re.compile(RE_XML_ILLEGAL)
-
+
class OvalGeneratorException (Exception):
pass
class DSAFormatException (OvalGeneratorException):
- code = 1
-
+ code = 1
+
def __createXMLElement (name, descr = None, attrs = {}):
- """
- Create XML element with text descr and attributes attrs
-
- Keyword arguments:
- name -- Name of XML element
- descr -- content of textNode (default None)
- attrs -- attributes of element (default {})
-
- Return created XML element
- """
-
- doc = xml.dom.minidom.Document ()
- element = doc.createElement (name)
-
- for (attr, value) in attrs.items():
- for match in regex.finditer(attr):
- attr = attr[:match.start()] + "?" + attr[match.end():]
- for match in regex.finditer(value):
- value = value[:match.start()] + "?" + value[match.end():]
- attribute = doc.createAttribute (attr.encode("utf8"))
- attribute.value = value.encode("utf8")
- element.attributes.setNamedItem (attribute)
-
- if descr != None:
- for match in regex.finditer(descr):
- descr = descr[:match.start()] + "?" + descr[match.end():]
- description = doc.createTextNode (descr.encode("utf8"))
- element.appendChild (description)
-
- return (element)
+ """
+ Create XML element with text descr and attributes attrs
+
+ Keyword arguments:
+ name -- Name of XML element
+ descr -- content of textNode (default None)
+ attrs -- attributes of element (default {})
+
+ Return created XML element
+ """
+
+ doc = xml.dom.minidom.Document ()
+ element = doc.createElement (name)
+
+ for (attr, value) in attrs.items():
+ for match in regex.finditer(attr):
+ attr = attr[:match.start()] + "?" + attr[match.end():]
+ for match in regex.finditer(value):
+ value = value[:match.start()] + "?" + value[match.end():]
+ attribute = doc.createAttribute (attr.encode("utf8"))
+ attribute.value = value.encode("utf8")
+ element.attributes.setNamedItem (attribute)
+
+ if descr != None:
+ for match in regex.finditer(descr):
+ descr = descr[:match.start()] + "?" + descr[match.end():]
+ description = doc.createTextNode (descr.encode("utf8"))
+ element.appendChild (description)
+
+ return (element)
namespace = "oval:org.debian.oval"
tests = __createXMLElement ("tests")
@@ -66,464 +66,464 @@
testsCurId = 1
objectsCurId = 1
statesCurId = 1
-
releaseArchHash = {"2.0" : 2, "2.1" : 4, "2.2": 6, "3.0" : 11, "3.1" : 12,
"4.0" : 11, "5.0": 12, "6.0": 11}
testsHash = {"arch" : {}, "release": {}, "obj": {}, "fileSte": {}, "unameSte"
: {}, "dpkgSte": {}}
#We need more info about alpha, arm, hppa, bmips, lmips
unameArchTable = {'i386' : 'i686', 'amd64' : 'x86-64', 'ia64' : 'ia64',
'powerpc' : 'ppc', 's390' : 's390x', 'm86k' : 'm86k'}
def __trimzero (val):
- value = val[:]
- while value[0] == "0":
- value = value[1:]
- return value
+ value = val[:]
+ while value[0] == "0":
+ value = value[1:]
+ return value
def __getNewId (type):
- """Generate new unique id for tests, objects or states
-
- Argument keqywords:
- type -- type of generated id test | object | state
-
- return Generate id like <namespace>:tst|obj|ste:<id>
- """
- global testsCurId, objectsCurId, statesCurId
-
- if type == "test":
- result = "%s:tst:%d" % (namespace, testsCurId)
- testsCurId += 1
-
- if type == "object":
- result = "%s:obj:%d" % (namespace, objectsCurId)
- objectsCurId += 1
-
- if type == "state":
- result = "%s:ste:%d" % (namespace, statesCurId)
- statesCurId += 1
-
- return (result)
+ """Generate new unique id for tests, objects or states
+
+ Argument keqywords:
+ type -- type of generated id test | object | state
+
+ return Generate id like <namespace>:tst|obj|ste:<id>
+ """
+ global testsCurId, objectsCurId, statesCurId
+
+ if type == "test":
+ result = "%s:tst:%d" % (namespace, testsCurId)
+ testsCurId += 1
+
+ if type == "object":
+ result = "%s:obj:%d" % (namespace, objectsCurId)
+ objectsCurId += 1
+
+ if type == "state":
+ result = "%s:ste:%d" % (namespace, statesCurId)
+ statesCurId += 1
+
+ return result
def __createOVALDpkginfoObject (name):
- """ Generate OVAL dpkginfo_object definition """
-
- if not testsHash["obj"].has_key(name):
- objectId = __getNewId ("object");
- object = __createXMLElement("dpkginfo_object",
- attrs={"id":objectId,
- "version":"1",
-
"xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#linux"})
- object.appendChild ( __createXMLElement ("name", name))
- objects.appendChild (object)
-
- testsHash["obj"][name] = objectId
-
- return (testsHash["obj"][name])
+ """ Generate OVAL dpkginfo_object definition """
+
+ if not testsHash["obj"].has_key(name):
+ objectId = __getNewId ("object");
+ object = __createXMLElement("dpkginfo_object",
+ attrs={"id":objectId,
+ "version":"1",
+
"xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#linux"})
+ object.appendChild ( __createXMLElement ("name", name))
+ objects.appendChild (object)
+
+ testsHash["obj"][name] = objectId
+
+ return (testsHash["obj"][name])
def __createOVALTextfilecontentObject (pattern, path = "/etc", filename =
"debian_version"):
- """ Generate OVAL textfilecontent_object definition """
- name = path + filename + pattern
-
- if not testsHash["obj"].has_key(name):
- objectId = __getNewId ("object");
- object = __createXMLElement("textfilecontent_object",
- attrs={"id":objectId,
- "version":"1",
-
"xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#independent"})
- object.appendChild ( __createXMLElement ("path", path))
- object.appendChild ( __createXMLElement ("filename", filename))
- object.appendChild ( __createXMLElement ("line", pattern,
attrs={"operation" : "pattern match"}))
- objects.appendChild (object)
-
- testsHash["obj"][name] = objectId
-
- return (testsHash["obj"][name])
+ """ Generate OVAL textfilecontent_object definition """
+ name = path + filename + pattern
+
+ if not testsHash["obj"].has_key(name):
+ objectId = __getNewId ("object");
+ object = __createXMLElement("textfilecontent_object",
+ attrs={"id":objectId,
+ "version":"1",
+
"xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#independent"})
+ object.appendChild ( __createXMLElement ("path", path))
+ object.appendChild ( __createXMLElement ("filename", filename))
+ object.appendChild ( __createXMLElement ("line", pattern,
attrs={"operation" : "pattern match"}))
+ objects.appendChild (object)
+
+ testsHash["obj"][name] = objectId
+
+ return (testsHash["obj"][name])
def __createOVALUnameObject ():
- """ Generate OVAL textfilecontent_object definition """
- name = "uname_object"
-
- if not testsHash["obj"].has_key(name):
- objectId = __getNewId ("object");
- object = __createXMLElement("uname_object",
- attrs={"id":objectId,
- "version":"1",
-
"xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#unix"})
- objects.appendChild (object)
-
- testsHash["obj"][name] = objectId
-
- return (testsHash["obj"][name])
+ """ Generate OVAL textfilecontent_object definition """
+ name = "uname_object"
+
+ if not testsHash["obj"].has_key(name):
+ objectId = __getNewId ("object");
+ object = __createXMLElement("uname_object",
+ attrs={"id":objectId,
+ "version":"1",
+
"xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#unix"})
+ objects.appendChild (object)
+
+ testsHash["obj"][name] = objectId
+
+ return (testsHash["obj"][name])
def __createOVALState (value, operation = "less than"):
- """ Generate OVAL state definition
-
- Use state hash for optimization of resulted XML
- """
- #TODO: Add arch state generation
- if not testsHash["dpkgSte"].has_key(operation) or not
testsHash["dpkgSte"][operation].has_key(value):
- stateId = __getNewId ("state")
-
- state = __createXMLElement("dpkginfo_state",
- attrs={"id":stateId,
- "version":"1",
-
"xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#linux"})
- state.appendChild ( __createXMLElement ("evr", "0:"+value,
-
{"datatype":"evr_string",
-
"operation":operation}))
- states.appendChild (state)
-
- testsHash["dpkgSte"][operation] = {value : stateId}
-
- return (testsHash["dpkgSte"][operation][value])
+ """ Generate OVAL state definition
+
+ Use state hash for optimization of resulted XML
+ """
+ #TODO: Add arch state generation
+ if not testsHash["dpkgSte"].has_key(operation) or not
testsHash["dpkgSte"][operation].has_key(value):
+ stateId = __getNewId ("state")
+
+ state = __createXMLElement("dpkginfo_state",
+ attrs={"id":stateId,
+ "version":"1",
+
"xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#linux"})
+ state.appendChild ( __createXMLElement ("evr", "0:"+value,
+ {"datatype":"evr_string",
+ "operation":operation}))
+ states.appendChild (state)
+
+ testsHash["dpkgSte"][operation] = {value : stateId}
+
+ return (testsHash["dpkgSte"][operation][value])
def __createOVALUnameState (field, value, operation = "equals"):
- """ Generate OVAL uname state definition
-
- Use unameArchTable to convert dsa arch to uname arch value
- """
-
- try:
- value = unameArchTable[value]
- except KeyError:
- pass
-
- #TODO: Add arch state generation
- if not testsHash["unameSte"].has_key(operation) or not
testsHash["unameSte"][operation].has_key(value):
- stateId = __getNewId ("state")
-
- state = __createXMLElement("uname_state",
- attrs={"id":stateId,
- "version":"1",
-
"xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#unix"})
- state.appendChild ( __createXMLElement (field, value,
-
{"operation":operation}))
- states.appendChild (state)
-
- testsHash["unameSte"][operation] = {value : stateId}
-
- return (testsHash["unameSte"][operation][value])
+ """ Generate OVAL uname state definition
+
+ Use unameArchTable to convert dsa arch to uname arch value
+ """
+
+ try:
+ value = unameArchTable[value]
+ except KeyError:
+ pass
+
+ #TODO: Add arch state generation
+ if not testsHash["unameSte"].has_key(operation) or not
testsHash["unameSte"][operation].has_key(value):
+ stateId = __getNewId ("state")
+
+ state = __createXMLElement("uname_state",
+ attrs={"id":stateId,
+ "version":"1",
+
"xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#unix"})
+ state.appendChild ( __createXMLElement (field, value,
+ {"operation":operation}))
+ states.appendChild (state)
+
+ testsHash["unameSte"][operation] = {value : stateId}
+
+ return (testsHash["unameSte"][operation][value])
def __createOVALTextfilecontentState (value, operation = "equals"):
- """ Generate OVAL state definition
-
- Use state hash for optimization of resulted XML
- """
- #TODO: Add arch state generation
- if not testsHash["fileSte"].has_key(operation) or not
testsHash["fileSte"][operation].has_key(value):
- stateId = __getNewId ("state")
-
- state = __createXMLElement("textfilecontent_state",
- attrs={"id":stateId,
- "version":"1",
-
"xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#independent"})
- state.appendChild ( __createXMLElement ("line", value,
-
{"operation":operation}))
- states.appendChild (state)
-
- testsHash["fileSte"][operation] = {value : stateId}
-
- return (testsHash["fileSte"][operation][value])
-
+ """ Generate OVAL state definition
+
+ Use state hash for optimization of resulted XML
+ """
+ #TODO: Add arch state generation
+ if not testsHash["fileSte"].has_key(operation) or not
testsHash["fileSte"][operation].has_key(value):
+ stateId = __getNewId ("state")
+
+ state = __createXMLElement("textfilecontent_state",
+ attrs={"id":stateId,
+ "version":"1",
+
"xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#independent"})
+ state.appendChild ( __createXMLElement ("line", value,
+ {"operation":operation}))
+ states.appendChild (state)
+
+ testsHash["fileSte"][operation] = {value : stateId}
+
+ return (testsHash["fileSte"][operation][value])
+
def __createDPKGTest(name, version):
- """ Generate OVAL DPKG test """
-
- ref = __getNewId ("test")
- test = __createXMLElement("dpkginfo_test",
- attrs={"id":ref,
- "version":"1",
- "check":"all",
- "check_existence":"at_least_one_exists",
- "comment":"%s is earlier than %s" % (name,
version),
-
"xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#linux";
- })
- test.appendChild ( __createXMLElement("object", attrs={"object_ref" :
__createOVALDpkginfoObject (name)}))
- test.appendChild ( __createXMLElement("state", attrs={"state_ref" :
__createOVALState (version)}))
- tests.appendChild(test)
+ """ Generate OVAL DPKG test """
+
+ ref = __getNewId ("test")
+ test = __createXMLElement("dpkginfo_test",
+ attrs={"id":ref,
+ "version":"1",
+ "check":"all",
+ "check_existence":"at_least_one_exists",
+ "comment":"%s is earlier than %s" % (name, version),
+
"xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#linux";
+ })
+ test.appendChild ( __createXMLElement("object", attrs={"object_ref" :
__createOVALDpkginfoObject (name)}))
+ test.appendChild ( __createXMLElement("state", attrs={"state_ref" :
__createOVALState (version)}))
+ tests.appendChild(test)
+
+ return (ref)
- return (ref)
-
def __createTest(testType, value):
- """ Generate OVAL test for release or architecture cases"""
-
- if not testsHash[testType].has_key(value):
- comment = None
-
- ref = __getNewId("test")
-
- if testType == "release":
- objectId = __createOVALTextfilecontentObject ("\d\.\d")
- comment = "Debian GNU/Linux %s is installed" % value
-
- test = __createXMLElement("textfilecontent_test",
- attrs={"id":ref,
- "version":"1",
- "check":"all",
- "check_existence":"at_least_one_exists",
- "comment":comment,
-
"xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#independent";
- })
- test.appendChild ( __createXMLElement("object",
attrs={"object_ref" : objectId}))
- test.appendChild ( __createXMLElement("state",
attrs={"state_ref" : __createOVALTextfilecontentState (value, "equals")}))
-
- else:
- objectId = __createOVALUnameObject ()
- comment = "Installed architecture is %s" % value
-
- test = __createXMLElement("uname_test",
- attrs={"id":ref,
- "version":"1",
- "check":"all",
- "check_existence":"at_least_one_exists",
- "comment":comment,
-
"xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#unix";
- })
- test.appendChild ( __createXMLElement("object",
attrs={"object_ref" : objectId}))
- if value != "all":
- test.appendChild ( __createXMLElement("state",
attrs={"state_ref" : __createOVALUnameState ("processor_type", value,
"equals")}))
-
- tests.appendChild(test)
-
- testsHash[testType][value] = ref
-
- return (testsHash[testType][value])
+ """ Generate OVAL test for release or architecture cases"""
+
+ if not testsHash[testType].has_key(value):
+ comment = None
+
+ ref = __getNewId("test")
+
+ if testType == "release":
+ objectId = __createOVALTextfilecontentObject ("\d\.\d")
+ comment = "Debian GNU/Linux %s is installed" % value
+
+ test = __createXMLElement("textfilecontent_test",
+ attrs={"id":ref,
+ "version":"1",
+ "check":"all",
+ "check_existence":"at_least_one_exists",
+ "comment":comment,
+
"xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#independent";
+ })
+ test.appendChild ( __createXMLElement("object",
attrs={"object_ref" : objectId}))
+ test.appendChild ( __createXMLElement("state", attrs={"state_ref"
: __createOVALTextfilecontentState (value, "equals")}))
+
+ else:
+ objectId = __createOVALUnameObject ()
+ comment = "Installed architecture is %s" % value
+
+ test = __createXMLElement("uname_test",
+ attrs={"id":ref,
+ "version":"1",
+ "check":"all",
+ "check_existence":"at_least_one_exists",
+ "comment":comment,
+
"xmlns":"http://oval.mitre.org/XMLSchema/oval-definitions-5#unix";
+ })
+ test.appendChild ( __createXMLElement("object",
attrs={"object_ref" : objectId}))
+ if value != "all":
+ test.appendChild ( __createXMLElement("state",
attrs={"state_ref" : __createOVALUnameState ("processor_type", value,
"equals")}))
+
+ tests.appendChild(test)
+
+ testsHash[testType][value] = ref
+
+ return (testsHash[testType][value])
def __createGeneratorHeader ():
- """
- Create OVAL definitions XML generator element.
-
- return xml.dom.minidom.Document with header information
- """
-
- doc = xml.dom.minidom.Document ()
- generator = doc.createElement ("generator")
-
- generator.appendChild ( __createXMLElement ("oval:product_name",
"Debian") )
- generator.appendChild ( __createXMLElement ("oval:schema_version",
"5.3") )
- generator.appendChild ( __createXMLElement ("oval:timestamp",
datetime.datetime.now().strftime ("%Y-%m-%dT%H:%M:%S.188-04:00")) )
+ """
+ Create OVAL definitions XML generator element.
+
+ return xml.dom.minidom.Document with header information
+ """
+
+ doc = xml.dom.minidom.Document ()
+ generator = doc.createElement ("generator")
+
+ generator.appendChild ( __createXMLElement ("oval:product_name", "Debian")
)
+ generator.appendChild ( __createXMLElement ("oval:schema_version", "5.3") )
+ generator.appendChild ( __createXMLElement ("oval:timestamp",
datetime.datetime.now().strftime ("%Y-%m-%dT%H:%M:%S.188-04:00")) )
- return (generator)
+ return (generator)
def createPlatformDefinition (release, data, dsa):
- """ Generate OVAL definitions for current release
-
- Generate full criteria tree for specified release. Tests,
states and objects
- stored in global dictionaries.
- Use differ module for otimize generated tree.
-
- Argument keywords:
- release -- Debian release
- data -- dict with information about packages
- dsa - DSA id
-
- return Generated XML fragment
- """
- #Raise exception if we receive too small data
- if len(data) == 0:
- logging.log(logging.WARNING, "DSA %s: Information of affected
platforms is not available." % dsa)
-
- softwareCriteria = __createXMLElement ("criteria", attrs = {"comment" :
"Release section", "operator" : "AND"})
- softwareCriteria.appendChild ( __createXMLElement ("criterion",
attrs={"test_ref" : __createTest("release", release), "comment" : "Debian %s is
installed" % release}))
-
- archCriteria = __createXMLElement ("criteria", attrs = {"comment" :
"Architecture section", "operator" : "OR"})
-
- # Handle architecture independed section
- if data.has_key ("all"):
- archIndepCriteria = __createXMLElement ("criteria",
attrs={"comment" : "Architecture independet section", "operator" : "AND"})
-
- archIndepCriteria.appendChild ( __createXMLElement
("criterion", attrs = {"test_ref" : __createTest("arch", "all"), "comment" :
"all architecture"}))
- #Build packages section only if we have more then one package
- if len (data["all"]) > 1:
- packageCriteria = __createXMLElement ("criteria",
attrs={"comment" : "Packages section", "operator" : "OR"})
- archIndepCriteria.appendChild (packageCriteria)
- else:
- packageCriteria = archIndepCriteria
-
- for pkg in data["all"].keys():
- packageCriteria.appendChild ( __createXMLElement
("criterion", attrs = {"test_ref" : __createDPKGTest(pkg, data["all"][pkg]),
"comment" : "%s DPKG is earlier than %s" % (pkg, data["all"][pkg])}))
-
- archCriteria.appendChild (archIndepCriteria)
-
- # Optimize packages tree in 2 stages
- diff = differ ()
- for i in range(2):
-
- if i == 0:
- dsaData = data
- else:
- dsaData = diff.getDiffer()
-
- diff.Clean()
- for (key, value) in dsaData.iteritems():
- if key != "all":
- diff.compareElement(key, value)
-
- eq = diff.getEqual()
- di = diff.getDiffer()
-
- # Generate XML for optimized packages
- if (len(eq)):
- if len(diff.getArchs()) != releaseArchHash[release]:
- archDependCriteria = __createXMLElement
("criteria", attrs={"comment" : "Architecture depended section", "operator" :
"AND"})
-
- supportedArchCriteria = __createXMLElement
("criteria", attrs={"comment" : "Supported architectures section", "operator" :
"OR"})
- for arch in diff.getArchs():
- supportedArchCriteria.appendChild (
__createXMLElement ("criterion", attrs = {"test_ref" : __createTest("arch",
arch), "comment" : "%s architecture" % arch}))
- archDependCriteria.appendChild
(supportedArchCriteria)
-
- packageCriteria = __createXMLElement ("criteria",
attrs={"comment" : "Packages section", "operator" : "OR"})
- for bpkg in eq.keys():
- packageCriteria.appendChild (
__createXMLElement ("criterion", attrs = {"test_ref" : __createDPKGTest(bpkg,
eq[bpkg]), "comment" : "%s DPKG is earlier than %s" % (bpkg, eq[bpkg])}))
-
- if len(diff.getArchs()) != releaseArchHash[release]:
- archDependCriteria.appendChild (packageCriteria)
- archCriteria.appendChild (archDependCriteria)
- else:
- archCriteria.appendChild (packageCriteria)
-
- # Generate XML for all other packages
- if len(di):
- archDependCriteria = __createXMLElement ("criteria",
attrs={"comment" : "Architecture depended section", "operator" : "AND"})
-
- for (key, value) in di.iteritems():
- supportedPlatformCriteria = __createXMLElement
("criteria", attrs={"comment" : "Supported platform section", "operator" :
"AND"})
- supportedPlatformCriteria.appendChild (
__createXMLElement ("criterion", attrs = {"test_ref" : __createTest("arch",
key), "comment" : "%s architecture" % key}))
-
- packageCriteria = __createXMLElement ("criteria",
attrs={"comment" : "Packages section", "operator" : "OR"})
-
- for bpkg in di[key].keys():
- packageCriteria.appendChild (
__createXMLElement ("criterion", attrs = {"test_ref" : __createDPKGTest(bpkg,
di[key][bpkg]), "comment" : "%s DPKG is earlier than %s" % (bpkg,
di[key][bpkg])}))
- supportedPlatformCriteria.appendChild
(packageCriteria)
-
- archDependCriteria.appendChild (supportedPlatformCriteria)
- archCriteria.appendChild (archDependCriteria)
-
- softwareCriteria.appendChild (archCriteria)
-
- return (softwareCriteria)
+ """ Generate OVAL definitions for current release
+
+ Generate full criteria tree for specified release. Tests, states and
objects
+ stored in global dictionaries.
+ Use differ module for optimize generated tree.
+
+ Argument keywords:
+ release -- Debian release
+ data -- dict with information about packages
+ dsa - DSA id
+
+ return Generated XML fragment
+ """
+ #Raise exception if we receive too small data
+ if len(data) == 0:
+ logging.log(logging.WARNING, "DSA %s: Information of affected
platforms is not available." % dsa)
+
+ softwareCriteria = __createXMLElement ("criteria", attrs = {"comment" :
"Release section", "operator" : "AND"})
+ softwareCriteria.appendChild ( __createXMLElement ("criterion",
attrs={"test_ref" : __createTest("release", release), "comment" : "Debian %s is
installed" % release}))
+
+ archCriteria = __createXMLElement ("criteria", attrs = {"comment" :
"Architecture section", "operator" : "OR"})
+
+ # Handle architecture independent section
+ if data.has_key ("all"):
+ archIndepCriteria = __createXMLElement ("criteria", attrs={"comment" :
"Architecture independent section", "operator" : "AND"})
+
+ archIndepCriteria.appendChild ( __createXMLElement ("criterion", attrs
= {"test_ref" : __createTest("arch", "all"), "comment" : "all architecture"}))
+ #Build packages section only if we have more then one package
+ if len (data["all"]) > 1:
+ packageCriteria = __createXMLElement ("criteria", attrs={"comment"
: "Packages section", "operator" : "OR"})
+ archIndepCriteria.appendChild (packageCriteria)
+ else:
+ packageCriteria = archIndepCriteria
+
+ for pkg in data["all"].keys():
+ packageCriteria.appendChild ( __createXMLElement ("criterion",
attrs = {"test_ref" : __createDPKGTest(pkg, data["all"][pkg]), "comment" : "%s
DPKG is earlier than %s" % (pkg, data["all"][pkg])}))
+
+ archCriteria.appendChild (archIndepCriteria)
+
+ # Optimize packages tree in 2 stages
+ diff = differ()
+ for i in range(2):
+
+ if i == 0:
+ dsaData = data
+ else:
+ dsaData = diff.getDiffer()
+
+ diff.Clean()
+ for (key, value) in dsaData.iteritems():
+ if key != "all":
+ diff.compareElement(key, value)
+
+ eq = diff.getEqual()
+ di = diff.getDiffer()
+
+ # Generate XML for optimized packages
+ if (len(eq)):
+ if len(diff.getArchs()) != releaseArchHash[release]:
+ archDependCriteria = __createXMLElement ("criteria",
attrs={"comment" : "Architecture depended section", "operator" : "AND"})
+
+ supportedArchCriteria = __createXMLElement ("criteria",
attrs={"comment" : "Supported architectures section", "operator" : "OR"})
+ for arch in diff.getArchs():
+ supportedArchCriteria.appendChild ( __createXMLElement
("criterion", attrs = {"test_ref" : __createTest("arch", arch), "comment" : "%s
architecture" % arch}))
+ archDependCriteria.appendChild (supportedArchCriteria)
+
+ packageCriteria = __createXMLElement ("criteria", attrs={"comment"
: "Packages section", "operator" : "OR"})
+ for bpkg in eq.keys():
+ packageCriteria.appendChild ( __createXMLElement ("criterion",
attrs = {"test_ref" : __createDPKGTest(bpkg, eq[bpkg]), "comment" : "%s DPKG is
earlier than %s" % (bpkg, eq[bpkg])}))
+
+ if len(diff.getArchs()) != releaseArchHash[release]:
+ archDependCriteria.appendChild (packageCriteria)
+ archCriteria.appendChild (archDependCriteria)
+ else:
+ archCriteria.appendChild (packageCriteria)
+
+ # Generate XML for all other packages
+ if len(di):
+ archDependCriteria = __createXMLElement ("criteria", attrs={"comment"
: "Architecture depended section", "operator" : "AND"})
+
+ for (key, value) in di.iteritems():
+ supportedPlatformCriteria = __createXMLElement ("criteria",
attrs={"comment" : "Supported platform section", "operator" : "AND"})
+ supportedPlatformCriteria.appendChild ( __createXMLElement
("criterion", attrs = {"test_ref" : __createTest("arch", key), "comment" : "%s
architecture" % key}))
+
+ packageCriteria = __createXMLElement ("criteria", attrs={"comment"
: "Packages section", "operator" : "OR"})
+
+ for bpkg in di[key].keys():
+ packageCriteria.appendChild ( __createXMLElement ("criterion",
attrs = {"test_ref" : __createDPKGTest(bpkg, di[key][bpkg]), "comment" : "%s
DPKG is earlier than %s" % (bpkg, di[key][bpkg])}))
+ supportedPlatformCriteria.appendChild (packageCriteria)
+
+ archDependCriteria.appendChild (supportedPlatformCriteria)
+ archCriteria.appendChild (archDependCriteria)
+
+ softwareCriteria.appendChild (archCriteria)
+
+ return (softwareCriteria)
def createDefinition (dsa, dsaref):
- """ Generate OVAL header of Definition tag
-
- Print general informaton about OVAL definition. Use
createPlatformDefinition for generate criteria
- sections for each affected release.
-
- Argument keywords:
- dsa -- DSA dentificator
- dsaref -- DSA parsed data
- """
- if not dsaref.has_key("release"):
- logging.log(logging.WARNING, "DSA %s: Release definition not
well formatted. Ignoring this DSA." % dsa)
- raise DSAFormatException
-
- if not dsaref.has_key("packages"):
- logging.log(logging.WARNING, "DSA %s: Package information
missed. Ignoring this DSA." % dsa)
- dsaref["packages"] = ""
-
- if not dsaref.has_key("description"):
- logging.log(logging.WARNING, "DSA %s: Description information
missed." % dsa)
- dsaref["description"] = ""
-
- if not dsaref.has_key("moreinfo"):
- logging.log(logging.WARNING, "DSA %s: Moreinfo information
missed." % dsa)
- dsaref["moreinfo"] = ""
-
- if not dsaref.has_key("secrefs"):
- logging.log(logging.WARNING, "DSA %s: Secrefs information
missed." % dsa)
- dsaref["secrefs"] = ""
-
- doc = xml.dom.minidom.Document ()
-
- ### Definition block: Metadata, Notes, Criteria
- ### TODO: Replace DSA id with unique id
- definition = __createXMLElement ("definition", attrs = {"id" :
"oval:org.debian:def:%s" % __trimzero(dsa), "version" : "1", "class" :
"vulnerability"})
-
- ### Definition : Metadata : title, affected, reference, description ###
- metadata = __createXMLElement ("metadata")
- metadata.appendChild (__createXMLElement ("title",
dsaref["description"]))
-
- ### Definition : Metadata : Affected : platform, product ###
- affected = __createXMLElement ("affected", attrs = {"family" : "unix"})
- for platform in dsaref["release"]:
- affected.appendChild ( __createXMLElement ("platform", "Debian
GNU/Linux %s" % platform))
- affected.appendChild ( __createXMLElement ("product",
dsaref.get("packages")))
-
- metadata.appendChild (affected)
- ### Definition : Metadata : Affected : END ###
-
- refpatern = re.compile (r'((CVE|CAN)-[\d-]+)')
- for ref in dsaref.get("secrefs").split(" "):
- result = refpatern.search(ref)
- if result:
- (ref_id, source) = result.groups()
- metadata.appendChild ( __createXMLElement ("reference",
attrs = {"source" : source, "ref_id" : ref_id, "ref_url" :
"http://cve.mitre.org/cgi-bin/cvename.cgi?name=%s"; % ref_id}) )
-
- #TODO: move this info to other place
- metadata.appendChild ( __createXMLElement ("description", "What
information can i put there?"))
- debianMetadata = __createXMLElement ("debian")
- if dsaref.has_key("date"):
- debianMetadata.appendChild ( __createXMLElement ("date",
dsaref["date"]) )
- debianMetadata.appendChild ( __createXMLElement ("moreinfo",
dsaref["moreinfo"]) )
- metadata.appendChild (debianMetadata)
- definition.appendChild ( metadata )
-
- ### Definition : Criteria ###
- if len(dsaref["release"]) > 1:
- #f we have more than one release - generate additional criteria
section
- platformCriteria = __createXMLElement ("criteria", attrs =
{"comment" : "Platform section", "operator" : "OR"})
- definition.appendChild (platformCriteria)
- else:
- platformCriteria = definition
-
- for platform in dsaref["release"]:
- data = dsaref["release"][platform]
- platformCriteria.appendChild
(createPlatformDefinition(platform, data, dsa))
-
- ### Definition : Criteria END ###
-
- return (definition)
-
-def createOVALDefinitions (dsaref):
- """ Generate XML OVAL definition tree for range of DSA
-
- Generate namespace section and use other functions to generate
definitions,
- tests, objects and states subsections.
-
- return -- Generated OVAL XML definition
- """
- doc = xml.dom.minidom.Document ()
-
- root = __createXMLElement ("oval_definitions",
- attrs= {
- "xsi:schemaLocation" :
"http://oval.mitre.org/XMLSchema/oval-definitions-5#independent
independent-definitions-schema.xsd
http://oval.mitre.org/XMLSchema/oval-definitions-5#linux
linux-definitions-schema.xsd
http://oval.mitre.org/XMLSchema/oval-definitions-5#unix
unix-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-definitions-5
oval-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-common-5
oval-common-schema.xsd",
- "xmlns:xsi" :
"http://www.w3.org/2001/XMLSchema-instance";,
- "xmlns:ind-def " :
"http://oval.mitre.org/XMLSchema/oval-definitions-5#independent";,
- "xmlns:linux-def" :
"http://oval.mitre.org/XMLSchema/oval-definitions-5#linux";,
- "xmlns:oval-def" :
"http://oval.mitre.org/XMLSchema/oval-definitions-5";,
- "xmlns:unix-def" :
"http://oval.mitre.org/XMLSchema/oval-definitions-5#unix";,
- "xmlns" :
"http://oval.mitre.org/XMLSchema/oval-definitions-5";,
- "xmlns:oval" :
"http://oval.mitre.org/XMLSchema/oval-common-5";
- }
- )
- doc.appendChild (root)
- root.appendChild ( __createGeneratorHeader () )
-
- definitions = doc.createElement ("definitions")
-
- keyids = dsaref.keys()
- keyids.sort()
- for dsa in keyids:
- try:
- definitions.appendChild (createDefinition(dsa,
dsaref[dsa]))
- except DSAFormatException:
- logging.log (logging.WARNING, "DSA %s: Bad data file.
Ignoring this DSA." % dsa)
-
- root.appendChild (definitions)
-
- root.appendChild(tests)
- root.appendChild(objects)
- root.appendChild(states)
+ """ Generate OVAL header of Definition tag
+
+ Print general information about OVAL definition. Use
createPlatformDefinition for generate criteria
+ sections for each affected release.
+
+ Argument keywords:
+ dsa -- DSA identifier
+ dsaref -- DSA parsed data
+ """
+
+ if not dsaref.has_key("release"):
+ logging.log(logging.WARNING, "DSA %s: Release definition not well
formatted. Ignoring this DSA." % dsa)
+ raise DSAFormatException
+
+ if not dsaref.has_key("packages"):
+ logging.log(logging.WARNING, "DSA %s: Package information missed.
Ignoring this DSA." % dsa)
+ dsaref["packages"] = ""
+
+ if not dsaref.has_key("description"):
+ logging.log(logging.WARNING, "DSA %s: Description information missed."
% dsa)
+ dsaref["description"] = ""
+
+ if not dsaref.has_key("moreinfo"):
+ logging.log(logging.WARNING, "DSA %s: Moreinfo information missed." %
dsa)
+ dsaref["moreinfo"] = ""
+
+ if not dsaref.has_key("secrefs"):
+ logging.log(logging.WARNING, "DSA %s: Secrefs information missed." %
dsa)
+ dsaref["secrefs"] = ""
+
+ doc = xml.dom.minidom.Document ()
+
+ ### Definition block: Metadata, Notes, Criteria
+ ### TODO: Replace DSA id with unique id
+ definition = __createXMLElement ("definition", attrs = {"id" :
"oval:org.debian:def:%s" % __trimzero(dsa), "version" : "1", "class" :
"vulnerability"})
+
+ ### Definition : Metadata : title, affected, reference, description ###
+ metadata = __createXMLElement ("metadata")
+ metadata.appendChild (__createXMLElement ("title", dsaref["description"]))
+
+ ### Definition : Metadata : Affected : platform, product ###
+ affected = __createXMLElement ("affected", attrs = {"family" : "unix"})
+ for platform in dsaref["release"]:
+ affected.appendChild ( __createXMLElement ("platform", "Debian
GNU/Linux %s" % platform))
+ affected.appendChild ( __createXMLElement ("product",
dsaref.get("packages")))
+
+ metadata.appendChild (affected)
+ ### Definition : Metadata : Affected : END ###
+
+ refpatern = re.compile (r'((CVE|CAN)-[\d-]+)')
+ for ref in dsaref.get("secrefs").split(" "):
+ result = refpatern.search(ref)
+ if result:
+ (ref_id, source) = result.groups()
+ metadata.appendChild ( __createXMLElement ("reference", attrs =
{"source" : source, "ref_id" : ref_id, "ref_url" :
"http://cve.mitre.org/cgi-bin/cvename.cgi?name=%s"; % ref_id}) )
+
+ #TODO: move this info to other place
+ metadata.appendChild ( __createXMLElement ("description", "What
information can i put there?"))
+ debianMetadata = __createXMLElement ("debian")
+ if dsaref.has_key("date"):
+ debianMetadata.appendChild ( __createXMLElement ("date",
dsaref["date"]) )
+ debianMetadata.appendChild ( __createXMLElement ("moreinfo",
dsaref["moreinfo"]) )
+ metadata.appendChild (debianMetadata)
+ definition.appendChild ( metadata )
+
+ ### Definition : Criteria ###
+ if len(dsaref["release"]) > 1:
+ #f we have more than one release - generate additional criteria section
+ platformCriteria = __createXMLElement ("criteria", attrs = {"comment"
: "Platform section", "operator" : "OR"})
+ definition.appendChild (platformCriteria)
+ else:
+ platformCriteria = definition
+
+ for platform in dsaref["release"]:
+ data = dsaref["release"][platform]
+ platformCriteria.appendChild (createPlatformDefinition(platform, data,
dsa))
+
+ ### Definition : Criteria END ###
+
+ return (definition)
+
+def createOVALDefinitions(dsaref):
+ """ Generate XML OVAL definition tree for range of DSA
+
+ Generate namespace section and use other functions to generate
definitions,
+ tests, objects and states subsections.
+
+ return -- Generated OVAL XML definition
+ """
+ doc = xml.dom.minidom.Document ()
+
+ root = __createXMLElement ("oval_definitions",
+ attrs= {
+ "xsi:schemaLocation" :
"http://oval.mitre.org/XMLSchema/oval-definitions-5#independent
independent-definitions-schema.xsd
http://oval.mitre.org/XMLSchema/oval-definitions-5#linux
linux-definitions-schema.xsd
http://oval.mitre.org/XMLSchema/oval-definitions-5#unix
unix-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-definitions-5
oval-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-common-5
oval-common-schema.xsd",
+ "xmlns:xsi" :
"http://www.w3.org/2001/XMLSchema-instance";,
+ "xmlns:ind-def " :
"http://oval.mitre.org/XMLSchema/oval-definitions-5#independent";,
+ "xmlns:linux-def" :
"http://oval.mitre.org/XMLSchema/oval-definitions-5#linux";,
+ "xmlns:oval-def" :
"http://oval.mitre.org/XMLSchema/oval-definitions-5";,
+ "xmlns:unix-def" :
"http://oval.mitre.org/XMLSchema/oval-definitions-5#unix";,
+ "xmlns" :
"http://oval.mitre.org/XMLSchema/oval-definitions-5";,
+ "xmlns:oval" :
"http://oval.mitre.org/XMLSchema/oval-common-5";
+ }
+ )
+ doc.appendChild (root)
+ root.appendChild ( __createGeneratorHeader () )
+
+ definitions = doc.createElement ("definitions")
+
+ keyids = dsaref.keys()
+ keyids.sort()
+ for dsa in keyids:
+ try:
+ definitions.appendChild (createDefinition(dsa, dsaref[dsa]))
+ except DSAFormatException:
+ logging.log (logging.WARNING, "DSA %s: Bad data file. Ignoring
this DSA." % dsa)
+
+ root.appendChild (definitions)
+
+ root.appendChild(tests)
+ root.appendChild(objects)
+ root.appendChild(states)
- return doc
+ return doc
def printOVALDefinitions (doc):
- if doc.getElementsByTagName("definitions")[0].hasChildNodes():
- print doc.toprettyxml()
+ if doc.getElementsByTagName("definitions")[0].hasChildNodes():
+ print doc.toprettyxml()
Binary files oval_old/oval/definition/generator.pyc and
oval/oval/definition/generator.pyc differ
Binary files oval_old/oval/definition/__init__.pyc and
oval/oval/definition/__init__.pyc differ
Binary files oval_old/oval/__init__.pyc and oval/oval/__init__.pyc differ
diff -urN oval_old/oval/parser/dsa.py oval/oval/parser/dsa.py
--- oval_old/oval/parser/dsa.py 2011-10-11 17:41:19.000000000 -0600
+++ oval/oval/parser/dsa.py 2015-11-13 09:34:04.843882000 -0700
@@ -9,6 +9,7 @@
# <isvulnerable>
# <fixed>
#
+# (c) 2015 Nicholas Luedtke
# (c) 2007 Pavel Vinogradov
# (c) 2004 Javier Fernandez-Sanguino
# Licensed under the GNU General Public License version 2.
@@ -17,105 +18,113 @@
import os
import logging
+
# Format of data files is:
-#<define-tag pagetitle>DSA-###-# PACKAGE</define-tag>
-#<define-tag report_date>yyyy-mm-dd</define-tag>
-#<define-tag secrefs>CAN|CVE-XXXX-XXXX</define-tag>
-#<define-tag packages>PACKAGE</define-tag>
-#<define-tag isvulnerable>yes|no</define-tag>
-#<define-tag fixed>yes|no</define-tag>
-def parseFile (path):
- """ Parse data file with information of Debian Security Advisories
+# <define-tag pagetitle>DSA-###-# PACKAGE</define-tag>
+# <define-tag report_date>yyyy-mm-dd</define-tag>
+# <define-tag secrefs>CAN|CVE-XXXX-XXXX</define-tag>
+# <define-tag packages>PACKAGE</define-tag>
+# <define-tag isvulnerable>yes|no</define-tag>
+# <define-tag fixed>yes|no</define-tag>
+def parseFile(path):
+ """ Parse data file with information of Debian Security Advisories
Keyword arguments:
path -- full path to data file
return list (dsa id, tags and packages data)"""
-
-
- data = {}
- deb_ver = None
- fdeb_ver = None
-
- filename = os.path.basename (path)
- patern = re.compile(r'dsa-(\d+)')
- result = patern.search(filename)
- if result:
- dsa = result.groups()[0]
- else:
- logging.log(logging.WARNING, "File %s does not look like a
proper DSA, not checking" % filename)
- return (None)
+ data = {}
+ deb_ver = None
+ fdeb_ver = None
+
+ filename = os.path.basename(path)
+
+ patern = re.compile(r'dsa-(\d+)')
+ result = patern.search(filename)
+
+ if result:
+ dsa = result.groups()[0]
+
+ else:
+ logging.log(logging.WARNING,
+ "File %s does not look like a proper DSA, not checking" %
filename)
+ return (None)
+
+ logging.log(logging.DEBUG, "Parsing DSA %s from file %s" % (dsa, filename))
+
+ dsaFile = open(path)
+
+ for line in dsaFile:
+ line = line.decode("ISO-8859-2")
+ datepatern = re.compile(r'report_date>([\d-]+)</define-tag>')
+ result = datepatern.search(line)
+ if result:
+ date = result.groups()[0]
+ normDate = lambda (date): "-".join(
+ [(len(p) > 1 and p or "0" + p) for p in date.split("-")])
+ data["date"] = normDate(date)
+
+ refspatern = re.compile(r'secrefs>(.*?)</define-tag>')
+ result = refspatern.search(line)
+ if result:
+ data["secrefs"] = result.groups()[0]
+ logging.log(logging.DEBUG,
+ "Extracted security references: " + data["secrefs"])
+
+ pakpatern = re.compile(r'packages>(.*?)</define-tag>')
+ result = pakpatern.search(line)
+ if result:
+ data["packages"] = result.groups()[0]
+
+ vulpatern = re.compile(r'isvulnerable>(.*?)</define-tag>')
+ result = vulpatern.search(line)
+ if result:
+ data["vulnarable"] = result.groups()[0]
+
+ fixpatern = re.compile(r'fixed>(.*?)</define-tag>')
+ result = fixpatern.search(line)
+ if result:
+ data["fixed"] = result.groups()[0]
+
+ versionpatern = re.compile(
+ r'<h3>Debian GNU/Linux (\d.\d) \((.*?)\)</h3>')
+ result = versionpatern.search(line)
+ if result:
+ fdeb_ver = result.groups()[0]
+
+ # Alternative format for data files
+ versionpatern = re.compile(r'affected_release>([\d\.]+)<')
+ result = versionpatern.search(line)
+ if result:
+ fdeb_ver = result.groups()[0]
+
+ if fdeb_ver:
+ deb_ver = fdeb_ver
+ fdeb_ver = None
+ if data.has_key("release"):
+ if data["release"].has_key(deb_ver):
+ logging.log(logging.WARNING,
+ "DSA %s: Found second files section for
release %s" % (
+ dsa, deb_ver))
+ else:
+ data["release"][deb_ver] = {}
+ else:
+ data["release"] = {deb_ver: {}}
+
+ # Binary packages are pushed into array
+ # Those are prepended by fileurls
+ # TODO: Packages do _NOT_ include epochs
+ # (that should be fixed)
+ if data.has_key("release") and deb_ver:
+ urlpatern = re.compile(r'fileurl
[\w:/.\-+]+/([\w\-.+~]+)\.deb[^i]')
+ result = urlpatern.search(line)
+ if result:
+ (package, version, architecture) =
result.groups()[0].split("_")
+
+ if data["release"][deb_ver].has_key(architecture):
+ data["release"][deb_ver][architecture][package] = version
+ else:
+ data["release"][deb_ver][architecture] = {package: version}
- logging.log (logging.DEBUG, "Parsing DSA %s from file %s" % (dsa,
filename))
-
- dsaFile = open(path)
-
- for line in dsaFile:
- line= line.decode ("ISO-8859-2")
- datepatern = re.compile (r'report_date>([\d-]+)</define-tag>')
- result = datepatern.search (line)
- if result:
- date = result.groups()[0]
- normDate = lambda (date): "-".join([(len(p) > 1 and p
or "0"+p) for p in date.split("-")])
- data["date"] = normDate(date)
-
- refspatern = re.compile (r'secrefs>(.*?)</define-tag>')
- result = refspatern.search (line)
- if result:
- data["secrefs"] = result.groups()[0]
- logging.log(logging.DEBUG, "Extracted security
references: " + data["secrefs"])
-
- pakpatern = re.compile (r'packages>(.*?)</define-tag>')
- result = pakpatern.search (line)
- if result:
- data["packages"] = result.groups()[0]
-
- vulpatern = re.compile (r'isvulnerable>(.*?)</define-tag>')
- result = vulpatern.search (line)
- if result:
- data["vulnarable"] = result.groups()[0]
-
- fixpatern = re.compile (r'fixed>(.*?)</define-tag>')
- result = fixpatern.search (line)
- if result:
- data["fixed"] = result.groups()[0]
-
- versionpatern = re.compile (r'<h3>Debian GNU/Linux (\d.\d)
\((.*?)\)</h3>')
- result = versionpatern.search (line)
- if result:
- fdeb_ver = result.groups()[0]
-
- # Alternative format for data files
- versionpatern = re.compile (r'affected_release>([\d\.]+)<')
- result = versionpatern.search (line)
- if result:
- fdeb_ver = result.groups()[0]
-
- if fdeb_ver:
- deb_ver = fdeb_ver
- fdeb_ver = None
- if data.has_key("release"):
- if data["release"].has_key(deb_ver):
- logging.log(logging.WARNING, "DSA %s:
Found second files section for release %s" % (dsa, deb_ver))
- else:
- data["release"][deb_ver] = {}
- else:
- data["release"] = {deb_ver: {}}
-
- # Binary packages are pushed into array
- # Those are prepended by fileurls
- # TODO: Packages do _NOT_ include epochs
- # (that should be fixed)
- if data.has_key("release") and deb_ver:
- urlpatern = re.compile (r'fileurl
[\w:/.\-+]+/([\w\-.+~]+)\.deb[^i]')
- result = urlpatern.search (line)
- if result:
- (package, version, architecture) =
result.groups()[0].split("_")
-
- if
data["release"][deb_ver].has_key(architecture):
-
data["release"][deb_ver][architecture][package] = version
- else:
- data["release"][deb_ver][architecture]
= {package : version}
-
- return (dsa, data)
+ return (dsa, data)
Binary files oval_old/oval/parser/dsa.pyc and oval/oval/parser/dsa.pyc differ
Binary files oval_old/oval/parser/__init__.pyc and
oval/oval/parser/__init__.pyc differ
diff -urN oval_old/oval/parser/wml.py oval/oval/parser/wml.py
--- oval_old/oval/parser/wml.py 2013-01-21 00:48:41.000000000 -0700
+++ oval/oval/parser/wml.py 2015-11-06 08:50:30.632363000 -0700
@@ -1,13 +1,14 @@
# -*- coding: utf-8 -*-
-# oval.parser.wml - module to parse descriptions of
+# oval.parser.wml - module to parse descriptions of
# Debian Security Advisories stored in wml format.
# Extrected tags:
# <description>
# <moreinfo>- Paragraphs before descriptions of
# each release status
#
-# (c) 2007 Pavel Vinogradov
-# (c) 2004 Javier Fernandez-Sanguino
+# (c) 2015 Nicholas Luedtke
+# (c) 2007 Pavel Vinogradov
+# (c) 2004 Javier Fernandez-Sanguino
# Licensed under the GNU General Public License version 2.
import re
@@ -15,76 +16,111 @@
import sys
import logging
+DEBIAN_VERSION = {"wheezy" : "7.0", "jessie" : "8.2", "stretch" : "9.0",
+ "sid" : "9.0", "etch" : "4.0", "squeeze":"6.0",
"lenny":"5.0"}
+
# Format of wml files is:
#<define-tag description>DESCRIPTION</define-tag>
#<define-tag moreinfo>Multiline information</define-tag>
def parseFile (path):
- """ Parse wml file with description of Debian Security Advisories
-
- Keyword arguments:
- path -- full path to wml file
-
- return list (dsa id, tags data)"""
-
- data = {}
- moreinfo = False
-
- filename = os.path.basename (path)
-
- patern = re.compile(r'dsa-(\d+)')
- result = patern.search(filename)
- if result:
- dsa = result.groups()[0]
- else:
- logging.log(logging.WARNING, "File %s does not look like a
proper DSA wml description, not checking" % filename)
- return (None)
-
- logging.log (logging.DEBUG, "Parsing information for DSA %s from wml
file %s" % (dsa, filename))
-
- try:
- wmlFile = open(path)
-
- for line in wmlFile:
- line= line.decode ("ISO-8859-2")
-
- descrpatern = re.compile
(r'description>(.*?)</define-tag>')
- result = descrpatern.search (line)
- if result:
- data["description"] = result.groups()[0]
- continue
-
- sinfopatern = re.compile (r'<define-tag moreinfo>(.*?)')
- result = sinfopatern.search (line)
- if result:
- moreinfo = True
- data["moreinfo"] = result.groups()[0]
- continue
-
- einfopatern = re.compile (r'</define-tag>')
- if moreinfo and einfopatern.search (line):
- data["moreinfo"] =
__parseMoreinfo(data["moreinfo"])
- moreinfo = False
- continue
-
- if moreinfo:
- data["moreinfo"] += line
- continue
-
- except IOError:
- logging.log (logging.ERROR, "Can't work with file %s" % path)
-
- return (dsa, data)
+ """ Parse wml file with description of Debian Security Advisories
+
+ Keyword arguments:
+ path -- full path to wml file
+
+ return list (dsa id, tags data)"""
+
+ data = {}
+ moreinfo = False
+ pack_ver = ""
+ deb_version = ""
+ releases = {}
+
+ filename = os.path.basename (path)
+
+ patern = re.compile(r'dsa-(\d+)')
+ result = patern.search(filename)
+ if result:
+ dsa = result.groups()[0]
+ else:
+ logging.log(logging.WARNING, "File %s does not look like a proper DSA
wml description, not checking" % filename)
+ return (None)
+
+ logging.log (logging.DEBUG, "Parsing information for DSA %s from wml file
%s" % (dsa, filename))
+
+ try:
+ wmlFile = open(path)
+
+ for line in wmlFile:
+ line= line.decode ("ISO-8859-2")
+
+ descrpatern = re.compile (r'description>(.*?)</define-tag>')
+ result = descrpatern.search (line)
+ if result:
+ data["description"] = result.groups()[0]
+ continue
+
+ sinfopatern = re.compile (r'<define-tag moreinfo>(.*?)')
+ result = sinfopatern.search (line)
+ if result:
+ moreinfo = True
+ data["moreinfo"] = result.groups()[0]
+ continue
+
+ einfopatern = re.compile (r'</define-tag>')
+ if moreinfo and einfopatern.search (line):
+ data["moreinfo"] = __parseMoreinfo(data["moreinfo"])
+ moreinfo = False
+ continue
+
+ if moreinfo:
+ data["moreinfo"] += line
+ #continue
+
+ dversion_pattern = re.compile(r'distribution \((.*?)\)')
+ result = dversion_pattern.search(line)
+ if result:
+ deb_version = result.groups()[0]
+
+ new_version_pattern = re.compile(r'version (.*?).</p>')
+ result = new_version_pattern.search(line)
+ if result and deb_version != "":
+ pack_ver = result.groups()[0]
+ releases.update({DEBIAN_VERSION[deb_version]: {u"all":
{grabPackName(path) : pack_ver}}})
+
+
+
+ except IOError:
+ logging.log (logging.ERROR, "Can't work with file %s" % path)
+
+ return dsa, data, releases
def __parseMoreinfo (info):
- """ Remove unnecessary information form moreinfo tag"""
+ """ Remove unnecessary information form moreinfo tag"""
+
+ p = re.compile ("<p>(.*?)</p>", re.DOTALL)
+ paragraphs = [m.groups()[0] for m in re.finditer(p, info)]
+ result = ""
+
+ for par in paragraphs:
+ if re.match(re.compile("For the .* distribution"), par):
+ break
+ result += "\n" + par
+
+ return result
+
+def grabPackName(path):
+ """
+ :param path: full path to wml file
+ :return: string: Package Name
+ """
- p = re.compile ("<p>(.*?)</p>", re.DOTALL)
- paragraphs = [m.groups()[0] for m in re.finditer(p, info)]
- result = ""
-
- for par in paragraphs:
- if re.match(re.compile("For the .* distribution"), par):
- break
- result += "\n" + par
-
- return result
+ try:
+ wmlFile = open(path)
+ package_name = re.compile (r'We recommend that you upgrade your (.*?)
packages')
+ for line in wmlFile:
+ result = package_name.search(line)
+ if result:
+ return result.groups()[0]
+ except IOError:
+ logging.log (logging.ERROR, "Can't work with file %s" % path)
Binary files oval_old/oval/parser/wml.pyc and oval/oval/parser/wml.pyc differ
diff -urN oval_old/parseDsa2Oval.py oval/parseDsa2Oval.py
--- oval_old/parseDsa2Oval.py 2011-10-12 15:14:55.000000000 -0600
+++ oval/parseDsa2Oval.py 2015-11-13 11:13:33.923838000 -0700
@@ -2,11 +2,11 @@
# -*- coding: utf-8 -*-
# Extracts the data DSA files and creates OVAL queries to
# be used with the OVAL query interpreter (see http://oval.mitre.org)
-
+# (c) 2015 Nicholas Luedtke
# (c) 2007 Pavel Vinogradov
# (c) 2004 Javier Fernandez-Sanguino
# Licensed under the GNU General Public License version 2.
-
+
import os
import sys
import getopt
@@ -18,89 +18,89 @@
dsaref = {}
+
def usage (prog = "parse-wml-oval.py"):
- """Print information about script flags and options"""
+ """Print information about script flags and options"""
- print """
-usage: %s [vh] [-d <directory>]
-\t-d\twhich directory use for dsa definition search
-\t-v\tverbose mode
-\t-h\tthis help
- """ % prog
-
-def printdsas (dsaref):
+ print """usage: %s [vh] [-d <directory>]\t-d\twhich directory use for
+ dsa definition search\t-v\tverbose mode\t-h\tthis help""" % prog
+
+def printdsas(dsaref):
""" Generate and print OVAL Definitions for collected DSA information """
-
+
ovalDefinitions = oval.definition.generator.createOVALDefinitions (dsaref)
oval.definition.generator.printOVALDefinitions (ovalDefinitions)
def parsedirs (directory, postfix, depth):
- """ Recursive search directory for DSA files contain postfix in their
names.
-
- For this files called oval.parser.dsa.parseFile() for
extracting DSA information.
- """
-
- if depth == 0:
- logging.log(logging.DEBUG, "Maximum depth reached at directory
" + directory)
- return (0)
+ """ Recursive search directory for DSA files contain postfix in their
names.
+ For this files called oval.parser.dsa.parseFile() for extracting DSA
+ information.
+ """
+
+ if depth == 0:
+ logging.log(logging.DEBUG, "Maximum depth reached at directory " +
directory)
+ return (0)
- for file in os.listdir (directory):
-
- path = "%s/%s" % (directory, file)
-
- logging.log (logging.DEBUG, "Checking %s (for %s at %s)" %
(file, postfix, depth))
+ for file in os.listdir (directory):
- if os.access(path, os.R_OK) and os.path.isdir (path) and not
os.path.islink (path) and file[0] != '.':
- logging.log(logging.DEBUG, "Entering directory " + path)
- parsedirs (path, postfix, depth-1)
+ path = "%s/%s" % (directory, file)
+ logging.log (logging.DEBUG, "Checking %s (for %s at %s)" % (file,
postfix, depth))
+ if os.access(path, os.R_OK) and os.path.isdir (path) and not
os.path.islink (path) and file[0] != '.':
+ logging.log(logging.DEBUG, "Entering directory " + path)
+ parsedirs (path, postfix, depth-1)
+
#Parse DSA data files
- if os.access(path, os.R_OK) and file.endswith(postfix) and
file[0] != '.' and file[0] != '#':
- result = dsa.parseFile (path)
- if result:
- if dsaref.has_key (result[0]):
- for (k, v) in result[1].iteritems():
- dsaref[result[0]][k] = v
- else:
- dsaref[result[0]] = result[1]
-
+ if os.access(path, os.R_OK) and file.endswith(postfix) and file[0] !=
'.' and file[0] != '#':
+ result = dsa.parseFile(path)
+ if result:
+ if dsaref.has_key(result[0]):
+ for (k, v) in result[1].iteritems():
+ dsaref[result[0]][k] = v
+ else:
+ dsaref[result[0]] = result[1]
+
#Parse DSA wml descriptions
- if os.access(path, os.R_OK) and file.endswith(".wml") and
file[0] != '.' and file[0] != '#':
- result = wml.parseFile(path)
- if result:
- if dsaref.has_key (result[0]):
- for (k, v) in result[1].iteritems():
- dsaref[result[0]][k] = v
- else:
- dsaref[result[0]] = result[1]
-
- return 0
+ if os.access(path, os.R_OK) and file.endswith(".wml") and file[0] !=
'.' and file[0] != '#':
+ result = wml.parseFile(path)
+ if result:
+ if dsaref.has_key(result[0]):
+ for (k, v) in result[1].iteritems():
+ dsaref[result[0]][k] = v
+ if not dsaref[result[0]].has_key("release"):
+ dsaref[result[0]]['release']=result[2]
+ else:
+ dsaref[result[0]] = result[1]
+ dsaref[result[0]]['release']=result[2]
+
+ return 0
if __name__ == "__main__":
-
+
# Parse cmd options with getopt
opts = {}
-
+
#By default we search dsa definitions from current directory, but -d
option override this
opts['-d'] = "./"
-
+
try:
opt, args = getopt.getopt (sys.argv[1:], 'vhd:')
except getopt.GetoptError:
usage ()
sys.exit(1)
-
+
for key, value in opt:
opts[key] = value
-
+
if opts.has_key ('-h'):
usage()
sys.exit(0)
-
+
if opts.has_key('-v'):
logging.basicConfig(level=logging.DEBUG)
-
+
logging.basicConfig(level=logging.WARNING)
-
+
parsedirs (opts['-d'], '.data', 2)
+
printdsas(dsaref)
diff -urN oval_old/parseJSON2Oval.py oval/parseJSON2Oval.py
--- oval_old/parseJSON2Oval.py 1969-12-31 17:00:00.000000000 -0700
+++ oval/parseJSON2Oval.py 2015-11-13 13:16:42.523783000 -0700
@@ -0,0 +1,191 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# Extracts the data from the security tracker and creates OVAL queries to
+# be used with the OVAL query interpreter (see http://oval.mitre.org)
+
+# (c) 2015 Nicholas Luedtke
+# Licensed under the GNU General Public License version 2.
+
+import os
+from subprocess import call
+import sys
+import logging
+import argparse
+import json
+from datetime import date
+import oval.definition.generator
+from oval.parser import dsa
+from oval.parser import wml
+
+
+dsaref = {}
+
+# TODO: these may need changed or reworked.
+DEBIAN_VERSION = {"wheezy" : "7.0", "jessie" : "8.2", "stretch" : "9.0",
+ "sid" : "9.0", "etch" : "4.0", "squeeze":"6.0",
"lenny":"5.0"}
+
+def usage (prog = "parse-wml-oval.py"):
+ """Print information about script flags and options"""
+
+ print """usage: %s [vh] [-d <directory>]\t-d\twhich directory use for
+ dsa definition search\t-v\tverbose mode\t-h\tthis help""" % prog
+
+
+def printdsas(dsaref):
+ """ Generate and print OVAL Definitions for collected DSA information """
+
+ ovalDefinitions = oval.definition.generator.createOVALDefinitions (dsaref)
+ oval.definition.generator.printOVALDefinitions (ovalDefinitions)
+
+
+def parsedirs (directory, postfix, depth):
+ """ Recursive search directory for DSA files contain postfix in their
names.
+ For this files called oval.parser.dsa.parseFile() for extracting DSA
+ information.
+ """
+ for file in os.listdir (directory):
+
+ path = "%s/%s" % (directory, file)
+ logging.log (logging.DEBUG, "Checking %s (for %s at %s)" % (file,
postfix, depth))
+
+ if os.access(path, os.R_OK) and os.path.isdir (path) and not
os.path.islink (path) and file[0] != '.':
+ logging.log(logging.DEBUG, "Entering directory " + path)
+ parsedirs (path, postfix, depth-1)
+
+ #Parse DSA data files
+ if os.access(path, os.R_OK) and file.endswith(postfix) and file[0] !=
'.' and file[0] != '#':
+ result = dsa.parseFile(path)
+ if result:
+ if dsaref.has_key(result[0]):
+ for (k, v) in result[1].iteritems():
+ dsaref[result[0]][k] = v
+ else:
+ dsaref[result[0]] = result[1]
+
+ #Parse DSA wml descriptions
+ if os.access(path, os.R_OK) and file.endswith(".wml") and file[0] !=
'.' and file[0] != '#':
+ result = wml.parseFile(path)
+ if result:
+ if dsaref.has_key(result[0]):
+ for (k, v) in result[1].iteritems():
+ dsaref[result[0]][k] = v
+ if not dsaref[result[0]].has_key("release"):
+ dsaref[result[0]]['release']=result[2]
+ else:
+ dsaref[result[0]] = result[1]
+ dsaref[result[0]]['release']=result[2]
+ return 0
+
+
+def parseJSON(json_data, id_num):
+ """
+ Parse the JSON data and extract information needed for OVAL definitions
+ :param id_num: int id number to start at for defintions
+ :param json_data: Json_Data
+ :return:
+ """
+ today = date.today()
+ logging.log(logging.DEBUG, "Start of JSON Parse.")
+ d_num = id_num
+ for package in json_data:
+ logging.log(logging.DEBUG, "Parsing package %s" % package)
+ for CVE in json_data[package]:
+ logging.log(logging.DEBUG, "Getting releases for %s" % CVE)
+ release = {}
+ for rel in json_data[package][CVE]['releases']:
+ if json_data[package][CVE]['releases'][rel]['status'] != \
+ 'resolved':
+ fixed_v = '0'
+ f_str = 'no'
+ else:
+ fixed_v =
json_data[package][CVE]['releases'][rel]['fixed_version']
+ f_str = 'yes'
+ release.update({DEBIAN_VERSION[rel]: {u'all': {
+ package: fixed_v}}})
+
+ dsaref.update({str(d_num): {"packages": package,
+ 'description': "",
+ 'vulnerable': "yes",
+ 'date': str(today.isoformat()),
+ 'fixed': f_str, 'moreinfo': "",
+ 'release': release, 'secrefs': CVE}})
+ logging.log(logging.DEBUG, "Created entry in dsaref %s" %
d_num)
+ d_num += 1
+
+
+def get_json_data(json_file):
+ """
+ Retrieves JSON formatted data from a file.
+ :param json_file:
+ :return: JSON data (dependent on the file loaded, usually a dictionary.)
+ """
+ logging.log(logging.DEBUG, "Extracting JSON file %s" % json_file)
+ with open(json_file, "r") as json_d:
+ d = json.load(json_d)
+ return d
+
+
+def main(args):
+ """
+ Main function for parseJSON2Oval.py
+ :param args:
+ :return:
+ """
+
+ if args['verbose']:
+ logging.basicConfig(level=logging.DEBUG)
+ else:
+ logging.basicConfig(level=logging.WARNING)
+
+ # unpack args
+
+ json_file = args['JSONfile']
+ temp_file = args['tmp']
+ id_num = args['id']
+
+ if json_file:
+ json_data = get_json_data(json_file)
+ else:
+ logging.log(logging.DEBUG, "Preparing to download JSONfile")
+ if os.path.isfile(temp_file):
+ logging.log(logging.WARNING, "Removing file %s" % temp_file)
+ os.remove(temp_file)
+ logging.log(logging.DEBUG, "Issuing wget for JSON file")
+ args = ['wget',
'https://security-tracker.debian.org/tracker/data/json',
+ '-O', temp_file]
+ call(args)
+ logging.log(logging.DEBUG, "File %s received" % temp_file)
+ json_data = get_json_data(temp_file)
+ if os.path.isfile(temp_file):
+ logging.log(logging.DEBUG, "Removing file %s" % temp_file)
+ os.remove(temp_file)
+
+ parseJSON(json_data, id_num)
+ #parsedirs (opts['-d'], '.data', 2)
+
+ printdsas(dsaref)
+
+if __name__ == "__main__":
+ PARSER = argparse.ArgumentParser(description='Generates oval definitions '
+ 'from the JSON file used to '
+ 'build the Debian Security '
+ 'Tracker.')
+ PARSER.add_argument('-v', '--verbose', help='Verbose Mode',
+ action="store_true")
+ PARSER.add_argument('-j', '--JSONfile', type=str,
+ help='Local JSON file to use. This will use a local '
+ 'copy of the JSON file instead of downloading
from'
+ ' it from the server. default=none', default=None)
+ PARSER.add_argument('-t', '--tmp', type=str,
+ help='Temporary file to download JSON file to.
Warning:'
+ ' if this file already exists it will be removed '
+ 'prior to downloading the JSON file. default= '
+ './DebSecTrackTMP.t',
default='./DebSecTrackTMP.t')
+ PARSER.add_argument('--id', type=int,
+ help='id number to start defintions at. default=100',
+ default=100)
+ ARGS = vars(PARSER.parse_args())
+ main(ARGS)
+
+
+
oval.tar.gz.sig
Description: oval.tar.gz.sig
diff.txt.sig
Description: diff.txt.sig
