Author: alfred
Date: Sun Nov 13 03:18:57 2011
New Revision: 227483
URL: http://svn.freebsd.org/changeset/base/227483
Log:
Utilize shell's IFS instead of forking ~6 processes to
handle splitting input files on a '|'. This greatly
reduces the time taken to process several databases
during the update process.
Additionally add some more debug logging.
Modified:
head/usr.sbin/portsnap/portsnap/portsnap.sh
Modified: head/usr.sbin/portsnap/portsnap/portsnap.sh
==============================================================================
--- head/usr.sbin/portsnap/portsnap/portsnap.sh Sun Nov 13 03:01:58 2011
(r227482)
+++ head/usr.sbin/portsnap/portsnap/portsnap.sh Sun Nov 13 03:18:57 2011
(r227483)
@@ -570,14 +570,16 @@ fetch_metadata_sanity() {
# Take a list of ${oldhash}|${newhash} and output a list of needed patches
fetch_make_patchlist() {
- grep -vE "^([0-9a-f]{64})\|\1$" |
- while read LINE; do
- X=`echo ${LINE} | cut -f 1 -d '|'`
- Y=`echo ${LINE} | cut -f 2 -d '|'`
- if [ -f "files/${Y}.gz" ]; then continue; fi
- if [ ! -f "files/${X}.gz" ]; then continue; fi
- echo "${LINE}"
+ IFS='|'
+ echo "" 1>${QUIETREDIR}
+ grep -vE "^([0-9a-f]{64})\|\1$" |
+ while read X Y; do
+ printf "Processing: $X $Y ...\r" 1>${QUIETREDIR}
+ if [ -f "files/${Y}.gz" -o ! -f "files/${X}.gz" ]; then
continue; fi
+ echo "${X}|${Y}"
done
+ echo "" 1>${QUIETREDIR}
+ IFS=
}
# Print user-friendly progress statistics
@@ -692,9 +694,8 @@ fetch_update() {
# Attempt to apply metadata patches
echo -n "Applying metadata patches... "
- while read LINE; do
- X=`echo ${LINE} | cut -f 1 -d '|'`
- Y=`echo ${LINE} | cut -f 2 -d '|'`
+ IFS='|'
+ while read X Y; do
if [ ! -f "${X}-${Y}.gz" ]; then continue; fi
gunzip -c < ${X}-${Y}.gz > diff
gunzip -c < files/${X}.gz > OLD
@@ -707,6 +708,7 @@ fetch_update() {
fi
rm -f diff OLD NEW ${X}-${Y}.gz ptmp
done < patchlist 2>${QUIETREDIR}
+ IFS=
echo "done."
# Update metadata without patches
@@ -724,16 +726,19 @@ fetch_update() {
2>${QUIETREDIR}
while read Y; do
+ echo -n "Verifying ${Y}... " 1>${QUIETREDIR}
if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then
mv ${Y}.gz files/${Y}.gz
else
echo "metadata is corrupt."
return 1
fi
+ echo "ok." 1>${QUIETREDIR}
done < filelist
echo "done."
# Extract the index
+ echo -n "Extracting index... " 1>${QUIETREDIR}
gunzip -c files/`look INDEX tINDEX.new |
cut -f 2 -d '|'`.gz > INDEX.new
fetch_index_sanity || return 1
@@ -754,8 +759,10 @@ fetch_update() {
fi
# Generate a list of wanted ports patches
+ echo -n "Generating list of wanted patches..." 1>${QUIETREDIR}
join -t '|' -o 1.2,2.2 INDEX INDEX.new |
fetch_make_patchlist > patchlist
+ echo " done." 1>${QUIETREDIR}
# Attempt to fetch ports patches
echo -n "Fetching `wc -l < patchlist | tr -d ' '` "
@@ -766,11 +773,18 @@ fetch_update() {
echo "done."
# Attempt to apply ports patches
- echo -n "Applying patches... "
- while read LINE; do
- X=`echo ${LINE} | cut -f 1 -d '|'`
- Y=`echo ${LINE} | cut -f 2 -d '|'`
- if [ ! -f "${X}-${Y}" ]; then continue; fi
+ PATCHCNT=`wc -l patchlist`
+ echo "Applying patches... "
+ IFS='|'
+ I=0
+ while read X Y; do
+ I=$(($I + 1))
+ F="${X}-${Y}"
+ if [ ! -f "${F}" ]; then
+ printf " Skipping ${F} (${I} of ${PATCHCNT}).\r"
+ continue;
+ fi
+ echo " Processing ${F}..." 1>${QUIETREDIR}
gunzip -c < files/${X}.gz > OLD
${BSPATCH} OLD NEW ${X}-${Y}
if [ `${SHA256} -q NEW` = ${Y} ]; then
@@ -779,6 +793,7 @@ fetch_update() {
fi
rm -f diff OLD NEW ${X}-${Y}
done < patchlist 2>${QUIETREDIR}
+ IFS=
echo "done."
# Update ports without patches
@@ -795,7 +810,10 @@ fetch_update() {
xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \
2>${QUIETREDIR}
+ I=0
while read Y; do
+ I=$(($I + 1))
+ printf " Processing ${Y} (${I} of ${PATCHCNT}).\r"
1>${QUIETREDIR}
if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then
mv ${Y}.gz files/${Y}.gz
else
@@ -877,6 +895,7 @@ extract_metadata() {
# Do the actual work involved in "extract"
extract_run() {
+ local IFS='|'
mkdir -p ${PORTSDIR} || return 1
if !
@@ -886,7 +905,7 @@ extract_run() {
grep -vE "${REFUSE}" ${WORKDIR}/INDEX
else
cat ${WORKDIR}/INDEX
- fi | tr '|' ' ' | while read FILE HASH; do
+ fi | while read FILE HASH; do
echo ${PORTSDIR}/${FILE}
if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then
echo "files/${HASH}.gz not found -- snapshot corrupt."
@@ -916,35 +935,8 @@ extract_run() {
extract_indices
}
-# Do the actual work involved in "update"
-update_run() {
- if ! [ -z "${INDEXONLY}" ]; then
- extract_indices >/dev/null || return 1
- return 0
- fi
-
- if sort ${WORKDIR}/INDEX |
- cmp -s ${PORTSDIR}/.portsnap.INDEX -; then
- echo "Ports tree is already up to date."
- return 0
- fi
-
-# If we are REFUSEing to touch certain directories, don't remove files
-# from those directories (even if they are out of date)
- echo -n "Removing old files and directories... "
- if ! [ -z "${REFUSE}" ]; then
- sort ${WORKDIR}/INDEX |
- comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' |
- grep -vE "${REFUSE}" |
- lam -s "${PORTSDIR}/" - |
- sed -e 's|/$||' | xargs rm -rf
- else
- sort ${WORKDIR}/INDEX |
- comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' |
- lam -s "${PORTSDIR}/" - |
- sed -e 's|/$||' | xargs rm -rf
- fi
- echo "done."
+update_run_exract() {
+ local IFS='|'
# Install new files
echo "Extracting new files:"
@@ -955,9 +947,7 @@ update_run() {
sort ${WORKDIR}/INDEX
fi |
comm -13 ${PORTSDIR}/.portsnap.INDEX - |
- while read LINE; do
- FILE=`echo ${LINE} | cut -f 1 -d '|'`
- HASH=`echo ${LINE} | cut -f 2 -d '|'`
+ while read FILE HASH; do
echo ${PORTSDIR}/${FILE}
if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then
echo "files/${HASH}.gz not found -- snapshot corrupt."
@@ -977,7 +967,39 @@ update_run() {
done; then
return 1
fi
+}
+
+# Do the actual work involved in "update"
+update_run() {
+ if ! [ -z "${INDEXONLY}" ]; then
+ extract_indices >/dev/null || return 1
+ return 0
+ fi
+
+ if sort ${WORKDIR}/INDEX |
+ cmp -s ${PORTSDIR}/.portsnap.INDEX -; then
+ echo "Ports tree is already up to date."
+ return 0
+ fi
+
+# If we are REFUSEing to touch certain directories, don't remove files
+# from those directories (even if they are out of date)
+ echo -n "Removing old files and directories... "
+ if ! [ -z "${REFUSE}" ]; then
+ sort ${WORKDIR}/INDEX |
+ comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' |
+ grep -vE "${REFUSE}" |
+ lam -s "${PORTSDIR}/" - |
+ sed -e 's|/$||' | xargs rm -rf
+ else
+ sort ${WORKDIR}/INDEX |
+ comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' |
+ lam -s "${PORTSDIR}/" - |
+ sed -e 's|/$||' | xargs rm -rf
+ fi
+ echo "done."
+ update_run_exract || return 1
extract_metadata
extract_indices
}
_______________________________________________
svn-src-head@freebsd.org mailing list
http://lists.freebsd.org/mailman/listinfo/svn-src-head
To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
