Author: gjb
Date: Mon Apr 27 19:41:40 2015
New Revision: 282111
URL: https://svnweb.freebsd.org/changeset/base/282111
Log:
MFC r280299, r280840, r280846, r280879, r280881, r280884, r280928,
r281783, r281809, r281832:
r280299 (cperciva):
When creating VM images, copy the contents of the created filesystem
into a new filesystem before packaging it into a disk image. This
prevents "remnants" of deleted files from showing up in the VM images,
and reduces their compressed size (by about 10% for the cloudware
images) as a result.
r280840 (cperciva):
Clean up filesystem unmounting in vmimage builds:
- Remove vm_umount_base function which is currently unused.
- Add umount_loop function which loops attempting to unmount one
filesystem.
- Replace calls to umount with calls to umount_loop.
- Don't attempt to unmount ${DESTDIR}/dev if it isn't mounted.
The looping is necessary because sometimes umount fails due to
filesystems being busy. The most common cause of such busyness is
periodic(8) jobs running `find / ...`.
r280846 (cperciva):
Improve check for whether ${DESTDIR}/dev is mounted.
r280879 (cperciva):
Add bits for building EC2 disk images.
r280881 (cperciva):
Partially revert r278118 now that the required logic for deciding
whether freebsd-update can be useful has moved into the
firstboot_freebsd_update script.
r280884 (cperciva):
Fix pkg(8) command line in vm_extra_pre_umount().
r280928 (cperciva):
Add code for creating an EC2 AMI.
r281783:
When building VM disk images, vm_copy_base() uses tar(1) to copy
the userland from one md(4)-mounted filesystem to a clean filesystem
to prevent remnants of files that were added and removed from
resulting in an unclean filesystem. When newfs(8) creates the first
filesystem with journaled soft-updates enabled, the /.sujournal file
in the new filesystem cannot be overwritten by the /.sujournal in the
original filesystem.
To avoid this particular error case, do not enable journaled
soft-updates when creating the md(4)-backed filesystems, and instead
use tunefs(8) to enable journaled soft-updates after the new
filesystem is populated in vm_copy_base().
While here, fix a long standing bug where the build environment /boot
files were used by mkimg(1) when creating the VM disk images by using
the files in .OBJDIR.
r281809:
Simplify variable expansion in attempt to fix the vm-image build.
r281832:
Fix locating the /boot files.
Sponsored by: The FreeBSD Foundation
Added:
stable/10/release/Makefile.ec2
- copied unchanged from r280928, head/release/Makefile.ec2
stable/10/release/tools/ec2.conf
- copied, changed from r280879, head/release/tools/ec2.conf
Modified:
stable/10/release/Makefile.vm
stable/10/release/scripts/mk-vmimage.sh
stable/10/release/tools/gce.conf
stable/10/release/tools/vmimage.subr
Directory Properties:
stable/10/ (props changed)
Copied: stable/10/release/Makefile.ec2 (from r280928, head/release/Makefile.ec2)
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ stable/10/release/Makefile.ec2 Mon Apr 27 19:41:40 2015
(r282111, copy of r280928, head/release/Makefile.ec2)
@@ -0,0 +1,44 @@
+#
+# $FreeBSD$
+#
+#
+# Makefile for creating an EC2 AMI from a disk image.
+#
+
+.if ${BRANCH} == "CURRENT" || ${BRANCH} == "STABLE"
+AMINAMESUFFIX!= date +-%Y-%m-%d
+.endif
+.if defined(EC2PUBLIC)
+PUBLISH= --public
+.endif
+
+ec2ami: cw-ec2
+.if !exists(/usr/local/bin/bsdec2-image-upload)
+ @echo "--------------------------------------------------------------"
+ @echo ">>> Creating EC2 AMIs requires bsdec2-image-upload"
+ @echo "--------------------------------------------------------------"
+ @false
+.endif
+.if !defined(AWSKEYFILE) || !exists(${AWSKEYFILE})
+ @echo "--------------------------------------------------------------"
+ @echo ">>> AWSKEYFILE must point at AWS keys for EC2 AMI creation"
+ @echo "--------------------------------------------------------------"
+ @false
+.endif
+.if !defined(AWSREGION)
+ @echo "--------------------------------------------------------------"
+ @echo ">>> AWSREGION must be specified EC2 AMI creation"
+ @echo "--------------------------------------------------------------"
+ @false
+.endif
+.if !defined(AWSBUCKET)
+ @echo "--------------------------------------------------------------"
+ @echo ">>> AWSBUCKET must be specified for EC2 AMI creation"
+ @echo "--------------------------------------------------------------"
+ @false
+.endif
+ /usr/local/bin/bsdec2-image-upload ${PUBLISH} \
+ ${.OBJDIR}/ec2.raw \
+ "${TYPE} ${REVISION}-${BRANCH}${AMINAMESUFFIX}" \
+ "${TYPE} ${REVISION}-${BRANCH}" \
+ ${AWSREGION} ${AWSBUCKET} ${AWSKEYFILE}
Modified: stable/10/release/Makefile.vm
==============================================================================
--- stable/10/release/Makefile.vm Mon Apr 27 19:18:51 2015
(r282110)
+++ stable/10/release/Makefile.vm Mon Apr 27 19:41:40 2015
(r282111)
@@ -16,11 +16,15 @@ QCOW2_DESC= Qemu, KVM disk image
RAW_DESC= Unformatted raw disk image
CLOUDWARE?= AZURE \
+ EC2 \
GCE \
OPENSTACK
AZURE_FORMAT= vhdf
AZURE_DESC= Microsoft Azure platform image
AZURE_DISK= ${OSRELEASE}.${AZURE_FORMAT}
+EC2_FORMAT= raw
+EC2_DESC= Amazon EC2 image
+EC2_DISK= ${OSRELEASE}.${EC2_FORMAT}
GCE_FORMAT= raw
GCE_DESC= Google Compute Engine image
GCE_DISK= disk.${GCE_FORMAT}
@@ -148,3 +152,5 @@ cloudware-install:
.if defined(WITH_CLOUDWARE) && !empty(WITH_CLOUDWARE) && !empty(CLOUDWARE)
${MAKE} -C ${.CURDIR} ${.MAKEFLAGS} ${CLOUDINSTALL}
.endif
+
+.include "${.CURDIR}/Makefile.ec2"
Modified: stable/10/release/scripts/mk-vmimage.sh
==============================================================================
--- stable/10/release/scripts/mk-vmimage.sh Mon Apr 27 19:18:51 2015
(r282110)
+++ stable/10/release/scripts/mk-vmimage.sh Mon Apr 27 19:41:40 2015
(r282111)
@@ -102,6 +102,7 @@ main() {
vm_extra_pre_umount
vm_extra_pkg_rmcache
cleanup
+ vm_copy_base
vm_create_disk || return 0
vm_extra_create_disk
Copied and modified: stable/10/release/tools/ec2.conf (from r280879,
head/release/tools/ec2.conf)
==============================================================================
--- head/release/tools/ec2.conf Tue Mar 31 01:28:33 2015 (r280879, copy
source)
+++ stable/10/release/tools/ec2.conf Mon Apr 27 19:41:40 2015
(r282111)
@@ -32,7 +32,7 @@ vm_extra_pre_umount() {
# catalogue and install or update pkg when the instance first
# launches, so these files would just be replaced anyway; removing
# them from the image allows it to boot faster.
- pkg -c ${DESTDIR} delete -f pkg
+ pkg -c ${DESTDIR} delete -f -y pkg
rm ${DESTDIR}/var/db/pkg/repo-*.sqlite
# The size of the EC2 root disk can be configured at instance launch
Modified: stable/10/release/tools/gce.conf
==============================================================================
--- stable/10/release/tools/gce.conf Mon Apr 27 19:18:51 2015
(r282110)
+++ stable/10/release/tools/gce.conf Mon Apr 27 19:41:40 2015
(r282111)
@@ -10,7 +10,7 @@ export VM_EXTRA_PACKAGES="firstboot-free
# Set to a list of third-party software to enable in rc.conf(5).
export VM_RC_LIST="google_accounts_manager ntpd sshd firstboot_growfs \
- firstboot_pkgs google_startup"
+ firstboot_pkgs firstboot_freebsd_update google_startup"
vm_extra_install_base() {
echo 'search google.internal' > ${DESTDIR}/etc/resolv.conf
@@ -23,13 +23,6 @@ vm_extra_pre_umount() {
dumpdev="AUTO"
ifconfig_DEFAULT="SYNCDHCP mtu 1460"
ntpd_sync_on_start="YES"
-case \$(uname -r) in
- *-BETA*|*-RC*|*-RELEASE*)
- firstboot_freebsd_update_enable="YES"
- ;;
- *)
- ;;
-esac
# need to fill in something here
#firstboot_pkgs_list=""
panicmail_autosubmit="YES"
Modified: stable/10/release/tools/vmimage.subr
==============================================================================
--- stable/10/release/tools/vmimage.subr Mon Apr 27 19:18:51 2015
(r282110)
+++ stable/10/release/tools/vmimage.subr Mon Apr 27 19:41:40 2015
(r282111)
@@ -14,17 +14,24 @@ write_partition_layout() {
SWAPOPT="-p freebsd-swap/swapfs::1G"
fi
+ _OBJDIR="$(make -C ${WORLDDIR} -V .OBJDIR)"
+ if [ -d "${_OBJDIR%%/usr/src}/${TARGET}.${TARGET_ARCH}" ]; then
+
BOOTFILES="/${_OBJDIR%%/usr/src}/${TARGET}.${TARGET_ARCH}/usr/src/sys/boot"
+ else
+ BOOTFILES="/${_OBJDIR}/sys/boot"
+ fi
+
case "${TARGET}:${TARGET_ARCH}" in
amd64:amd64 | i386:i386)
- mkimg -s gpt -b /boot/pmbr \
- -p freebsd-boot/bootfs:=/boot/gptboot \
+ mkimg -s gpt -b ${BOOTFILES}/i386/pmbr/pmbr \
+ -p
freebsd-boot/bootfs:=${BOOTFILES}/i386/gptboot/gptboot \
${SWAPOPT} \
-p freebsd-ufs/rootfs:=${VMBASE} \
-o ${VMIMAGE}
;;
powerpc:powerpc*)
mkimg -s apm \
- -p apple-boot/bootfs:=/boot/boot1.hfs \
+ -p
apple-boot/bootfs:=${BOOTFILES}/powerpc/boot1.chrp/boot1.hfs \
${SWAPOPT} \
-p freebsd-ufs/rootfs:=${VMBASE} \
-o ${VMIMAGE}
@@ -45,8 +52,10 @@ err() {
}
cleanup() {
- umount ${DESTDIR}/dev 2>/dev/null
- umount ${DESTDIR}
+ if [ -c "${DESTDIR}/dev/null" ]; then
+ umount_loop ${DESTDIR}/dev 2>/dev/null
+ fi
+ umount_loop ${DESTDIR}
if [ ! -z "${mddev}" ]; then
mdconfig -d -u ${mddev}
fi
@@ -61,12 +70,42 @@ vm_create_base() {
mkdir -p ${DESTDIR}
truncate -s ${VMSIZE} ${VMBASE}
mddev=$(mdconfig -f ${VMBASE})
- newfs -j /dev/${mddev}
+ newfs /dev/${mddev}
mount /dev/${mddev} ${DESTDIR}
return 0
}
+vm_copy_base() {
+ # Creates a new UFS root filesystem and copies the contents of the
+ # current root filesystem into it. This produces a "clean" disk
+ # image without any remnants of files which were created temporarily
+ # during image-creation and have since been deleted (e.g., downloaded
+ # package archives).
+
+ mkdir -p ${DESTDIR}/old
+ mdold=$(mdconfig -f ${VMBASE})
+ mount /dev/${mdold} ${DESTDIR}/old
+
+ truncate -s ${VMSIZE} ${VMBASE}.tmp
+ mkdir -p ${DESTDIR}/new
+ mdnew=$(mdconfig -f ${VMBASE}.tmp)
+ newfs /dev/${mdnew}
+ mount /dev/${mdnew} ${DESTDIR}/new
+
+ tar -cf- -C ${DESTDIR}/old . | tar -xUf- -C ${DESTDIR}/new
+
+ umount_loop /dev/${mdold}
+ rmdir ${DESTDIR}/old
+ mdconfig -d -u ${mdold}
+
+ umount_loop /dev/${mdnew}
+ rmdir ${DESTDIR}/new
+ tunefs -j enable /dev/${mdnew}
+ mdconfig -d -u ${mdnew}
+ mv ${VMBASE}.tmp ${VMBASE}
+}
+
vm_install_base() {
# Installs the FreeBSD userland/kernel to the virtual machine disk.
@@ -88,7 +127,7 @@ vm_install_base() {
mount -t devfs devfs ${DESTDIR}/dev
chroot ${DESTDIR} /usr/bin/newaliases
chroot ${DESTDIR} /etc/rc.d/ldconfig forcestart
- umount ${DESTDIR}/dev
+ umount_loop ${DESTDIR}/dev
cp /etc/resolv.conf ${DESTDIR}/etc/resolv.conf
@@ -123,7 +162,7 @@ vm_extra_install_packages() {
/usr/sbin/pkg bootstrap -y
chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes \
/usr/sbin/pkg install -y ${VM_EXTRA_PACKAGES}
- umount ${DESTDIR}/dev
+ umount_loop ${DESTDIR}/dev
return 0
}
@@ -152,16 +191,17 @@ vm_extra_pkg_rmcache() {
return 0
}
-vm_umount_base() {
+umount_loop() {
+ DIR=$1
i=0
sync
- while ! umount ${DESTDIR}/dev ${DESTDIR}; do
+ while ! umount ${DIR}; do
i=$(( $i + 1 ))
if [ $i -ge 10 ]; then
# This should never happen. But, it has happened.
- msg="Cannot umount(8) ${DESTDIR}\n"
- msg="${msg}Something has gone horribly wrong."
- err "${msg}"
+ echo "Cannot umount(8) ${DIR}"
+ echo "Something has gone horribly wrong."
+ return 1
fi
sleep 1
done
_______________________________________________
svn-src-stable-10@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-stable-10
To unsubscribe, send any mail to "svn-src-stable-10-unsubscr...@freebsd.org"
