Revision: 7743
http://gar.svn.sourceforge.net/gar/?rev=7743&view=rev
Author: wahwah
Date: 2009-12-24 17:16:17 +0000 (Thu, 24 Dec 2009)
Log Message:
-----------
mGAR v2-checkpkg: sqlite3 backend for the system-wide pkgmap cache, much faster
Modified Paths:
--------------
csw/mgar/gar/v2-checkpkg/bin/checkpkg
csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg-dummy.py
csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg-libs.py
csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg.py
Modified: csw/mgar/gar/v2-checkpkg/bin/checkpkg
===================================================================
--- csw/mgar/gar/v2-checkpkg/bin/checkpkg 2009-12-24 16:36:19 UTC (rev
7742)
+++ csw/mgar/gar/v2-checkpkg/bin/checkpkg 2009-12-24 17:16:17 UTC (rev
7743)
@@ -26,7 +26,6 @@
LOCAL_ARCH=`uname -p`
-
# always print out a warning message. (to stderr)
# exit script, if quit_on_warn set
@@ -67,6 +66,12 @@
exit 1
}
+debugmsg() {
+ if [[ "${DEBUG}" != "" ]]; then
+ print "DEBUG: $*" > /dev/fd/2
+ fi
+}
+
set_variables_for_individual_package_check() {
f=$1
file $f |sed 's/^.*://' |grep gzip >/dev/null
@@ -566,33 +571,38 @@
set_variables_for_individual_package_check "$f"
echo "Running the experimental plugin infrastructure."
+test_suite_ok=1
checkpkg_scriptname=`basename $0`
checkpkg_basedir=${0%/${checkpkg_scriptname}}
plugindir=${checkpkg_basedir}/checkpkg.d
if [[ "${DEBUG}" != "" ]]; then
extra_options="--debug"
fi
-echo "plugindir: '$plugindir'"
+debugmsg "plugindir: '$plugindir'"
if [[ -d "$plugindir" ]]; then
# echo plugin dir exists
for plugin in "${plugindir}"/checkpkg-*; do
if [[ -x "${plugin}" ]]; then
- echo "Executing: ${plugin} $extra_options -e
\"${EXTRACTDIR}\" ${pkgnames}"
+ debugmsg "Executing: ${plugin} $extra_options -e
\"${EXTRACTDIR}\" ${pkgnames}"
${plugin} $extra_options -e "${EXTRACTDIR}" ${pkgnames}
if [[ "$?" -ne 0 ]]; then
- errmsg "Plugin ${plugin} has returned an error."
+ print "TEST: ${plugin} [FAIL]"
+ test_suite_ok=0
+ else
+ print "TEST: ${plugin} [OK]"
fi
else
- echo "'${plugin}' is not executable"
+ debugmsg "'${plugin}' is not executable"
fi
done
else
- echo plugin dir does not exist
+ debugmsg "plugin dir does not exist"
fi
-# echo "Profiling."
-# python -m cProfile gar/bin/checkpkg.d/checkpkg-libs.py --debug -e
"${EXTRACTDIR}" ${pkgnames}
-# End of plugin section
+if [[ ${test_suite_ok} -ne 1 ]]; then
+ errmsg "One or more tests have failed."
+fi
+
print ""
# Cleaning up after all packages
Modified: csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg-dummy.py
===================================================================
--- csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg-dummy.py 2009-12-24
16:36:19 UTC (rev 7742)
+++ csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg-dummy.py 2009-12-24
17:16:17 UTC (rev 7743)
@@ -3,6 +3,7 @@
import checkpkg
import os.path
+import logging
def main():
options, args = checkpkg.GetOptions()
@@ -12,8 +13,8 @@
pkgpath = os.path.join(options.extractdir, pkgname)
if not os.path.isdir(pkgpath):
raise checkpkg.PackageError("The package directory doesn't exist: %s" %
pkgpath)
- print ("Dummy plugin says the package %s is extracted to %s"
- % (pkgname, options.extractdir))
+ logging.debug("Dummy plugin says the package %s is extracted to %s",
+ pkgname, options.extractdir)
if __name__ == '__main__':
Modified: csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg-libs.py
===================================================================
--- csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg-libs.py 2009-12-24
16:36:19 UTC (rev 7742)
+++ csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg-libs.py 2009-12-24
17:16:17 UTC (rev 7743)
@@ -121,18 +121,21 @@
pkgs_by_soname[soname] = pkgname
# A shared object dependency/provisioning report, plus checking.
- for soname in needed_sonames:
- if soname in needed_sonames_by_binary:
- print "%s is provided by the package itself" % soname
- elif soname in lines_by_soname:
- print ("%s is required by %s and provided by %s"
- % (soname,
- binaries_by_soname[soname],
- repr(pkgs_by_soname[soname])))
- else:
- print ("%s is required by %s, but we don't know what provides it."
- % (soname, binaries_by_soname[soname]))
- result_ok = False
+ if needed_sonames:
+ print "Analysis of sonames needed by the package set:"
+ for soname in needed_sonames:
+ if soname in needed_sonames_by_binary:
+ print "%s is provided by the package itself" % soname
+ elif soname in lines_by_soname:
+ print ("%s is required by %s and provided by %s"
+ % (soname,
+ binaries_by_soname[soname],
+ repr(pkgs_by_soname[soname])))
+ else:
+ print ("%s is required by %s, but we don't know what provides it."
+ % (soname, binaries_by_soname[soname]))
+ result_ok = False
+ print
dependent_pkgs = {}
for checker in checkers:
@@ -152,12 +155,12 @@
logging.warn("%s not found in needed_sonames_by_binary (%s)",
binary, needed_sonames_by_binary.keys())
declared_dependencies_set = set(declared_dependencies)
- print "You can consider including the following packages in the
dependencies:"
- for dep_pkgname in
sorted(so_dependencies.difference(declared_dependencies_set)):
- print " ", dep_pkgname,
- if dep_pkgname.startswith("SUNW"):
- print "(it's safe to ignore this one)",
- print
+ missing_deps = so_dependencies.difference(declared_dependencies_set)
+ if missing_deps:
+ print "SUGGESTION: you may want to add some or all of the following as
depends:"
+ print " (Feel free to ignore SUNW or SPRO packages)"
+ for dep_pkgname in sorted(missing_deps):
+ print ">", dep_pkgname
surplus_dependencies =
declared_dependencies_set.difference(so_dependencies)
surplus_dependencies =
surplus_dependencies.difference(TYPICAL_DEPENDENCIES)
Modified: csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg.py
===================================================================
--- csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg.py 2009-12-24 16:36:19 UTC
(rev 7742)
+++ csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg.py 2009-12-24 17:16:17 UTC
(rev 7743)
@@ -10,6 +10,7 @@
import subprocess
import cPickle
import re
+import sqlite3
SYSTEM_PKGMAP = "/var/sadm/install/contents"
WS_RE = re.compile(r"\s+")
@@ -89,26 +90,38 @@
class SystemPkgmap(object):
"""A class to hold and manipulate the /var/sadm/install/contents file."""
- PICKLE_NAME = "var-sadm-install-contents.pickle"
STOP_PKGS = ["SUNWbcp", "SUNWowbcp", "SUNWucb"]
CHECKPKG_DIR = ".checkpkg"
+ SQLITE3_DBNAME = "var-sadm-install-contents-cache"
def __init__(self):
"""There is no need to re-parse it each time.
Read it slowly the first time and cache it for later."""
self.checkpkg_dir = os.path.join(os.environ["HOME"], self.CHECKPKG_DIR)
- self.pickle_path = os.path.join(self.checkpkg_dir, self.PICKLE_NAME)
- if os.path.exists(self.pickle_path):
- logging.info("Unpickling %s, this can take up to 30s.", self.pickle_path)
- pickle_fd = open(self.pickle_path, "r")
- self.pkmap_lines_by_basename = cPickle.load(pickle_fd)
- pickle_fd.close()
+ self.db_path = os.path.join(self.checkpkg_dir, self.SQLITE3_DBNAME)
+ if os.path.exists(self.db_path):
+ logging.debug("Connecting to the %s database.", self.db_path)
+ self.conn = sqlite3.connect(self.db_path)
else:
- # The original checkpkg code to port is in the comments.
- #
+ logging.info("Building a cache of /var/sadm/install/contents.")
+ if not os.path.exists(self.checkpkg_dir):
+ logging.debug("Creating %s", self.checkpkg_dir)
+ os.mkdir(self.checkpkg_dir)
+ self.conn = sqlite3.connect(self.db_path)
+ c = self.conn.cursor()
+ c.execute("""
+ CREATE TABLE systempkgmap (
+ id INTEGER PRIMARY KEY,
+ basename TEXT,
+ path TEXT,
+ line TEXT
+ );
+ """)
+
# egrep -v 'SUNWbcp|SUNWowbcp|SUNWucb' /var/sadm/install/contents |
# fgrep -f $EXTRACTDIR/liblist >$EXTRACTDIR/shortcatalog
+
system_pkgmap_fd = open(SYSTEM_PKGMAP, "r")
stop_re = re.compile("(%s)" % "|".join(self.STOP_PKGS))
@@ -117,29 +130,23 @@
# soname - {<path1>: <line1>, <path2>: <line2>, ...}
logging.debug("Building in-memory data structure for the %s file",
SYSTEM_PKGMAP)
- pkmap_lines_by_basename = {}
for line in system_pkgmap_fd:
if stop_re.search(line):
continue
fields = re.split(WS_RE, line)
pkgmap_entry_path = fields[0].split("=")[0]
pkgmap_entry_dir, pkgmap_entry_base_name =
os.path.split(pkgmap_entry_path)
- if pkgmap_entry_base_name not in pkmap_lines_by_basename:
- pkmap_lines_by_basename[pkgmap_entry_base_name] = {}
- pkmap_lines_by_basename[pkgmap_entry_base_name][pkgmap_entry_dir] =
line
- logging.debug("The data structure contains %s files",
- len(pkmap_lines_by_basename))
- self.pkmap_lines_by_basename = pkmap_lines_by_basename
- if not os.path.exists(self.checkpkg_dir):
- logging.debug("Creating %s", self.checkpkg_dir)
- os.mkdir(self.checkpkg_dir)
- logging.debug("Pickling to %s", self.pickle_path)
- pickle_fd = open(self.pickle_path, "w")
- cPickle.dump(self.pkmap_lines_by_basename, pickle_fd)
- pickle_fd.close()
+ sql = "INSERT INTO systempkgmap (basename, path, line) VALUES (?, ?,
?);"
+ c.execute(sql, (pkgmap_entry_base_name, pkgmap_entry_dir, line))
+ logging.info("Creating an index.")
+ sql = "CREATE INDEX basename_idx ON systempkgmap(basename);"
+ self.conn.execute(sql)
def GetPkgmapLineByBasename(self, filename):
- if filename in self.pkmap_lines_by_basename:
- return self.pkmap_lines_by_basename[filename]
- else:
- raise KeyError, "%s not found in self.pkmap_lines_by_basename" % filename
+ sql = "SELECT path, line FROM systempkgmap WHERE basename = ?;"
+ c = self.conn.cursor()
+ c.execute(sql, [filename])
+ lines = {}
+ for row in c:
+ lines[row[0]] = row[1]
+ return lines
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
_______________________________________________
devel mailing list
devel@lists.opencsw.org
https://lists.opencsw.org/mailman/listinfo/devel
