Revision: 19675
http://gar.svn.sourceforge.net/gar/?rev=19675&view=rev
Author: wahwah
Date: 2012-11-12 15:57:09 +0000 (Mon, 12 Nov 2012)
Log Message:
-----------
integrate_catalogs: Use curl instead of pkgdb
It's way faster. Currently, most of the time is spent on the server side.
Package insertios are significantly slower than deletions.
Modified Paths:
--------------
csw/mgar/gar/v2/lib/python/integrate_catalogs.py
Modified: csw/mgar/gar/v2/lib/python/integrate_catalogs.py
===================================================================
--- csw/mgar/gar/v2/lib/python/integrate_catalogs.py 2012-11-12 15:55:20 UTC
(rev 19674)
+++ csw/mgar/gar/v2/lib/python/integrate_catalogs.py 2012-11-12 15:57:09 UTC
(rev 19675)
@@ -23,28 +23,47 @@
import rest
import sys
import urllib2
+import re
CATALOG_MOD_TMPL = """#!/bin/bash
# Catalog modification (not integration yet): $catrel_from -> $catrel_to
# Generated by $prog
+
+if ! grep buildfarm ~/.netrc
+then
+ touch ~/.netrc
+ chmod 0600 ~/.netrc
+ echo >> ~/.netrc \
+ "machine buildfarm.opencsw.org login \${LOGNAME} password \$(cat
/etc/opt/csw/releases/auth/\${LOGNAME})"
+fi
+
set -x
-PKGDB=bin/pkgdb
+readonly CURL="curl --netrc"
+readonly REST_URL=http://buildfarm.opencsw.org/releases/
+function _add_to_cat {
+ \${CURL} -X PUT \${REST_URL}catalogs/$1/$2/$3/$4/
+}
+
+function _del_from_cat {
+ \${CURL} -X DELETE \${REST_URL}catalogs/$1/$2/$3/$4/
+}
+
#for catalogname in $sorted($diffs_by_catalogname):
#if "new_pkgs" in $diffs_by_catalogname[$catalogname]:
function new_pkg_$catalogname {
#for arch, osrel, new_pkg in $diffs_by_catalogname[$catalogname]["new_pkgs"]:
# adding $new_pkg["basename"]
- \${PKGDB} add-to-cat $osrel $arch $catrel_to $new_pkg["md5_sum"]
+ _add_to_cat $catrel_to $arch $osrel $new_pkg["md5_sum"]
#end for
}
function undo_new_pkg_$catalogname {
#for arch, osrel, new_pkg in $diffs_by_catalogname[$catalogname]["new_pkgs"]:
- # adding $new_pkg["basename"]
- \${PKGDB} del-from-cat $osrel $arch $catrel_to $new_pkg["md5_sum"]
+ # UNDO adding $new_pkg["basename"]
+ _del_from_cat $catrel_to $arch $osrel $new_pkg["md5_sum"]
#end for
}
#end if
@@ -52,13 +71,13 @@
function remove_pkg_$catalogname {
#for arch, osrel, rem_pkg in
$diffs_by_catalogname[$catalogname]["removed_pkgs"]:
# removing $rem_pkg["basename"]
- \${PKGDB} del-from-cat $osrel $arch $catrel_to $rem_pkg["md5_sum"]
+ _del_from_cat $catrel_to $arch $osrel $rem_pkg["md5_sum"]
#end for
}
function undo_remove_pkg_$catalogname {
#for arch, osrel, rem_pkg in
$diffs_by_catalogname[$catalogname]["removed_pkgs"]:
- # removing $rem_pkg["basename"]
- \${PKGDB} add-to-cat $osrel $arch $catrel_to $rem_pkg["md5_sum"]
+ # UNDO removing $rem_pkg["basename"]
+ _add_to_cat $catrel_to $arch $osrel $rem_pkg["md5_sum"]
#end for
}
#end if
@@ -75,15 +94,15 @@
# WARNING: DOWNGRADE
#end if
# $catalogname $up_pkg_pair["direction"] from
$up_pkg_pair["from"]["version"] to $up_pkg_pair["to"]["version"]
- \${PKGDB} del-from-cat $osrel $arch $catrel_to
$up_pkg_pair["from"]["md5_sum"]
- \${PKGDB} add-to-cat $osrel $arch $catrel_to $up_pkg_pair["to"]["md5_sum"]
+ _del_from_cat $catrel_to $arch $osrel $up_pkg_pair["from"]["md5_sum"]
+ _add_to_cat $catrel_to $arch $osrel $up_pkg_pair["to"]["md5_sum"]
#end for
}
function undo_upgrade_$catalogname {
#for arch, osrel, up_pkg_pair in
$diffs_by_catalogname[$catalogname]["updated_pkgs"]:
# UNDO of $catalogname $up_pkg_pair["direction"] from
$up_pkg_pair["from"]["version"] to $up_pkg_pair["to"]["version"]
- \${PKGDB} del-from-cat $osrel $arch $catrel_to $up_pkg_pair["to"]["md5_sum"]
- \${PKGDB} add-to-cat $osrel $arch $catrel_to $up_pkg_pair["from"]["md5_sum"]
+ _del_from_cat $catrel_to $arch $osrel $up_pkg_pair["to"]["md5_sum"]
+ _add_to_cat $catrel_to $arch $osrel $up_pkg_pair["from"]["md5_sum"]
#end for
}
#end if
@@ -116,7 +135,6 @@
#end for
"""
-
class Error(Exception):
"""Generic error."""
@@ -202,10 +220,12 @@
# By passing the catalogs (as arguments) in reverse order, we get
# packages to be updated in new_pkgs, and so forth.
for pkg in new_pkgs:
- catalogname_d = diffs_by_catalogname.setdefault(pkg["catalogname"], {})
+ catalogname_d = diffs_by_catalogname.setdefault(
+ (pkg["catalogname"]), {})
catalogname_d.setdefault("new_pkgs", []).append((arch, osrel, pkg))
for pkg in removed_pkgs:
- catalogname_d = diffs_by_catalogname.setdefault(pkg["catalogname"], {})
+ catalogname_d = diffs_by_catalogname.setdefault(
+ (pkg["catalogname"]), {})
catalogname_d.setdefault("removed_pkgs", []).append((arch, osrel, pkg))
for pkg_pair in updated_pkgs:
update_decision_by_type = {
@@ -215,7 +235,8 @@
if (update_decision_by_type[pkg_pair["type"]]
and (pkg_pair["direction"] == "upgrade" or include_downgrades)):
pkg = pkg_pair["from"]
- catalogname_d = diffs_by_catalogname.setdefault(pkg["catalogname"],
{})
+ catalogname_d = diffs_by_catalogname.setdefault(
+ (pkg["catalogname"]), {})
catalogname_d.setdefault("updated_pkgs", []).append((arch, osrel,
pkg_pair))
return diffs_by_catalogname
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
_______________________________________________
devel mailing list
devel@lists.opencsw.org
https://lists.opencsw.org/mailman/listinfo/devel
