Attached are a couple of rough alpha quality patches.
They implement what I have described rather verbosely on fedora-livecd-list and
in bug 248082.
The short story is the fedora7 livecd installer works by copying a 4G ext3 image
to the destination rootfs, and then resizing to maximal size.
The attached patches improve the speed of this step by 10-30% (for cdrom vs fast
livecd-iso-to-disk'd usbflash install media respectively).
This is accomplished because the 4G image actually only holds 2G of data. The
patch to livecd-creator, when invoked with --turbo-liveinst, will create a small
(25kb) binary delta file on the livecd. The patch to anaconda, will detect the
presence of the file, and if it is there, use it with devicemapper to create a
2G ext3 image, which can naturally be copied to the destination volume more
quickly. (I.e. 2G of zeros don't get written to disk).
In addition to improving installation speed, this also results in rootfs volumes
of 2.1G->3.9G being supported.
This patch is less than polished and elegant. But I'm hoping it might make it
into f8t1. Please review, and give suggestions for improvements. I'll plan on
trying to clean it up in whatever ways seem best over the next couple of days.
These are patches against current livecd git and anaconda cvs snaps. I think
they will work (very limited testing so far), but mainly I post them to get
eyeballs and brains looking at the code and the idea. No doubt within 24-48
hours I will post a more respectable looking patchset that I hope more people
might be willing to test.
(note, the cleanupDeleted patch I just posted to livecd-tools is included here,
as it lays the foundation for this... as well as reclaiming lots (5-15+%) of
wasted space on the livecd)
-dmc
diff -Naur livecd.git.20070723/creator/livecd-creator livecd/creator/livecd-creator
--- livecd.git.20070723/creator/livecd-creator 2007-07-23 02:39:15.000000000 -0500
+++ livecd/creator/livecd-creator 2007-07-23 02:39:50.000000000 -0500
@@ -163,7 +163,8 @@
def _formatFilesystem(self):
rc = subprocess.call(["/sbin/mkfs.ext3", "-F", "-L", self.fslabel,
- "-m", "1", self.lofile])
+ "-m", "1", "-b", "4096", self.lofile,
+ "%d" % ( self.size * 1024L * 1024L / 4096L )])
if rc != 0:
raise MountError("Error creating ext3 filesystem")
rc = subprocess.call(["/sbin/tune2fs", "-c0", "-i0", "-Odir_index",
@@ -286,7 +287,7 @@
return self.runTransaction(cb)
class InstallationTarget:
- def __init__(self, repos, packages, epackages, groups, fs_label, skip_compression, skip_prelink,tmpdir):
+ def __init__(self, repos, packages, epackages, groups, fs_label, skip_compression, turbo_liveinst, skip_prelink,tmpdir):
self.ayum = None
self.repos = repos
self.packages = packages
@@ -294,6 +295,7 @@
self.groups = groups
self.fs_label = fs_label
self.skip_compression = skip_compression
+ self.turbo_liveinst = turbo_liveinst
self.skip_prelink = skip_prelink
self.tmpdir = tmpdir
@@ -301,6 +303,7 @@
self.instloop = None
self.bindmounts = []
self.ksparser = None
+ self.minsizekb = 0
def parse(self, kscfg):
ksversion = pykickstart.version.makeVersion()
@@ -468,6 +471,10 @@
b.umount()
if self.instloop:
+ (xxbsize, xxfrsize, xxblocks, xxbfree,
+ xxbavail, xxfiles, xxffree, xxfavail,
+ xxflag, xxnamemax) = os.statvfs(self.build_dir + "/install_root")
+ print >> sys.stderr, "Installation target uncompressed data size is %d MB" % ( ( xxfrsize * xxblocks - xxbsize * xxbfree ) / ( 1024L * 1024L ) )
self.instloop.cleanup()
self.instloop = None
@@ -908,15 +915,139 @@
"""create compressed squashfs file system"""
if not self.skip_compression:
# FIXME: mksquashfs segfaults if PWD isn't set in the environment
- subprocess.call(["/sbin/mksquashfs", "os.img", "sysroot",
- "../out/squashfs.img"],
- cwd="%s/data" %(self.build_dir,),
- env={"PWD": "%s/data" %(self.build_dir,)})
+ if self.turbo_liveinst:
+ subprocess.call(["/sbin/mksquashfs", "os.img", "sysroot",
+ "osmin.tar", "../out/squashfs.img"],
+ cwd="%s/data" %(self.build_dir,),
+ env={"PWD": "%s/data" %(self.build_dir,)})
+ else:
+ subprocess.call(["/sbin/mksquashfs", "os.img", "sysroot",
+ "../out/squashfs.img"],
+ cwd="%s/data" %(self.build_dir,),
+ env={"PWD": "%s/data" %(self.build_dir,)})
else:
shutil.move("%s/data/os.img" %(self.build_dir,),
"%s/out/ext3fs.img" %(self.build_dir,))
+ if self.turbo_liveinst:
+ shutil.move("%s/data/osmin.tar" %(self.build_dir,),
+ "%s/out/ext3fsm.tar" %(self.build_dir,))
+
+ #
+ # cleanupDeleted removes unused data from the sparse ext3 os image file.
+ # The process involves: resize2fs-to-minimal, truncation,
+ # resize2fs-to-uncompressed-size (with implicit resparsification)
+ #
+ def cleanupDeleted(self):
+ # e2fsck -f -y is required by resize2fs
+ subprocess.call(["/sbin/e2fsck", "-f", "-y", "os.img"],
+ cwd="%s/data" %(self.build_dir,),
+ env={"PWD": "%s/data" %(self.build_dir,)})
+
+
+ (xxmode,xxino,xxdev,xxnlink,xxuid,xxgid,xxsize,xxatime,
+ xxmtime,xxctime) = os.stat(self.build_dir + "/data/os.img")
+ kbsize = xxsize / 1024L
+
+ # resize2fs doesn't have any kind of minimal setting, so use
+ # a binary search to get it to minimal size.
+ # TODO: if a --verbose/debug flag exists, don't devnull the output
+ FNULL = os.open('/dev/null', os.O_WRONLY)
+ size_top = kbsize / 4L
+ size_bot = 0
+ while size_top != (size_bot + 1):
+ trysize = (size_bot + ((size_top - size_bot) / 2)) * 4
+ resize2fs_retval = subprocess.call(["/sbin/resize2fs", "os.img",
+ "%dK" %(trysize,)],
+ cwd="%s/data" %(self.build_dir,),
+ env={"PWD": "%s/data" %(self.build_dir,)},
+ stdout=FNULL,
+ stderr=FNULL)
+
+ if not resize2fs_retval:
+ size_top = (size_bot + ((size_top - size_bot) / 2))
+ else:
+ size_bot = (size_bot + ((size_top - size_bot) / 2))
+
+
+ os.close(FNULL)
+
+ print >> sys.stderr, "Installation target minimized to %dK" % (size_top * 4)
+ # save minimized size for reuse by turboLiveInst
+ self.minsizekb = size_top * 4L
+
+ # truncate the unused excess portion of the sparse file
+ fd = os.open("%s/data/os.img" %(self.build_dir,), os.O_WRONLY )
+ os.ftruncate(fd, size_top * 4096L)
+ os.close(fd)
+
+
+ # resize back to uncompressed-size
+ resize2fs_retval = subprocess.call(["/sbin/resize2fs", "os.img",
+ "%dK" %(kbsize,)],
+ cwd="%s/data" %(self.build_dir,),
+ env={"PWD": "%s/data" %(self.build_dir,)})
+
+ #
+ # turboLiveInst: generates an osmin overlay file to sit alongside
+ # os.img. liveinst may then detect the existence of
+ # osmin, and use it to create a minimized os.img
+ # which can be installed more quickly, and to smaller
+ # destination volumes.
+ #
+ def turboLiveInst(self, image_size):
+ # create the sparse file for the minimized overlay
+ fd = os.open("%s/data/osmin" %(self.build_dir,),
+ os.O_WRONLY | os.O_CREAT)
+ off = long(16L * 1024L * 1024L)
+ os.lseek(fd, off, 0)
+ os.write(fd, '\x00')
+ os.close(fd)
+
+ # associate os image with loop device
+ osloop = LoopbackMount("%s/data/os.img" %(self.build_dir,),
+ "not_going_to_actually_get_mounted")
+ osloop.loopsetup()
+
+ # associate overlay with loop device
+ minloop = LoopbackMount("%s/data/osmin" %(self.build_dir,),
+ "not_going_to_actually_get_mounted")
+ minloop.loopsetup()
+
+ # create a snapshot device
+ rc = subprocess.call(["/sbin/dmsetup",
+ "--table",
+ "0 %d snapshot %s %s p 8"
+ %(image_size * 1024L * 2L,
+ osloop.loopdev, minloop.loopdev),
+ "create",
+ "livecd-creator-%d" %(os.getpid(),) ])
+ if rc != 0:
+ raise InstallationError("Could not create turboLiveInst snapshot device")
+
+ # resize snapshot device back to minimal (self.minsize)
+ rc = subprocess.call(["/sbin/resize2fs",
+ "/dev/mapper/livecd-creator-%d" %(os.getpid(),),
+ "%dK" %(self.minsizekb,)])
+
+ # tear down snapshot and loop devices
+ rc = subprocess.call(["/sbin/dmsetup", "remove",
+ "livecd-creator-%d" %(os.getpid(),) ])
+ if rc != 0:
+ raise InstallationError("Could not remove turboLiveInst snapshot device")
+ osloop.lounsetup()
+ minloop.lounsetup()
+
+ # use tar to unsparse osming because squashfs is fairly inefficient
+ # when it comes to sparse files.
+ rc = subprocess.call(["/bin/tar", "--sparse", "-cvf", "osmin.tar",
+ "osmin"],
+ cwd="%s/data" %(self.build_dir,),
+ env={"PWD": "%s/data" %(self.build_dir,)})
+
+ os.unlink(self.build_dir + "/data/osmin")
def package(self):
+
self.createSquashFS()
self.createIso()
@@ -931,6 +1062,8 @@
[--fslabel=<label>]
[--skip-compression]
[--uncompressed-size=<size-in-MB>]
+ [--ignore-deleted]
+ [--turbo-liveinst]
[--shell]
[--tmpdir=<tmpdir>]
@@ -944,6 +1077,9 @@
--skip-compression : Don't compress the image
--prelink : Prelink the image
--uncompressed-size : Size of uncompressed fs in MB (default: 4096)
+ --ignore-deleted : Don't run resize2fs to clean up wasted blocks
+ --turbo-liveinst : Create a small minimized fs image overlay file
+ primarily used by liveinst to improve speed
--shell : Start a shell in the chroot for post-configuration
--tmpdir : Temporary directory to use (default: /var/tmp)
@@ -972,8 +1108,10 @@
self.base_on = None
self.kscfg = None
self.skip_compression = False
+ self.turbo_liveinst = False
self.skip_prelink = True
self.uncompressed_size = 4096
+ self.ignore_deleted = False
self.give_shell = False
self.tmpdir = "/var/tmp"
@@ -983,7 +1121,10 @@
["help", "repo=", "base-on=", "package=",
"exclude-package=", "fslabel=", "config=",
"skip-compression", "uncompressed-size=",
- "shell", "no-prelink", "prelink","tmpdir="])
+ "ignore-deleted", "turbo-liveinst",
+ "shell", "no-prelink", "prelink",
+ "tmpdir="])
+
except getopt.GetoptError, msg:
raise Usage(msg)
@@ -1007,6 +1148,12 @@
if o in ("-u", "--uncompressed-size"):
options.uncompressed_size = int(a)
continue
+ if o in ("--ignore-deleted",):
+ options.ignore_deleted = True
+ continue
+ if o in ("--turbo-liveinst",):
+ options.turbo_liveinst = True
+ continue
if o in ("-c", "--config"):
options.kscfg = a
if not os.path.isfile(options.kscfg):
@@ -1049,6 +1196,9 @@
if not options.kscfg and not options.repos:
raise Usage("No repositories specified")
+ if options.turbo_liveinst and options.ignore_deleted:
+ raise Usage("turbo-liveinst can not be used with ignore-deleted")
+
return options
def main():
@@ -1076,6 +1226,7 @@
options.groups,
options.fs_label,
options.skip_compression,
+ options.turbo_liveinst,
options.skip_prelink,
options.tmpdir)
@@ -1091,9 +1242,17 @@
print "----------------------------------"
target.launchShell()
+
target.unmount()
+ if not options.ignore_deleted:
+ target.cleanupDeleted()
+
+ if options.turbo_liveinst:
+ target.turboLiveInst(options.uncompressed_size)
+
target.package()
+
except InstallationError, e:
print >> sys.stderr, "Error creating Live CD : %s" % e
target.teardown()
diff -Naur livecd.git.20070723/creator/mayflower livecd/creator/mayflower
--- livecd.git.20070723/creator/mayflower 2007-07-23 02:39:15.000000000 -0500
+++ livecd/creator/mayflower 2007-07-23 02:39:58.000000000 -0500
@@ -609,8 +609,10 @@
#
if [ -e /sysroot/LiveOS/ext3fs.img ]; then
EXT3FS="/sysroot/LiveOS/ext3fs.img"
+ EXT3MIN="/sysroot/LiveOS/ext3fsm.tar"
elif [ -e /sysroot/ext3fs.img ] ; then
EXT3FS="/sysroot/ext3fs.img"
+ EXT3MIN="/sysroot/ext3fsm.tar"
fi
if [ -n "\$EXT3FS" ] ; then
@@ -627,6 +629,10 @@
modprobe loop max_loop=128
modprobe dm_snapshot
+ if [ -f \$EXT3MIN ] ; then
+ losetup /dev/loop118 \$EXT3MIN
+ fi
+
losetup /dev/loop121 \$EXT3FS
umount -l /sysroot
@@ -670,6 +676,10 @@
mkdir -p /squashfs
mount -n -t squashfs -o ro /dev/loop120 /squashfs
+ if [ -f /squashfs/osmin.tar ] ; then
+ losetup /dev/loop118 /squashfs/osmin.tar
+ fi
+
losetup /dev/loop121 /squashfs/os.img
umount -l /squashfs
diff -Naur anaconda.cvs.20070723/livecd.py anaconda/livecd.py
--- anaconda.cvs.20070723/livecd.py 2007-07-16 14:45:33.000000000 -0500
+++ anaconda/livecd.py 2007-07-23 02:37:04.000000000 -0500
@@ -128,16 +128,23 @@
return self.osimg
def getLiveSizeMB(self):
- lnk = os.readlink(self.osimg)
- if lnk[0] != "/":
- lnk = os.path.join(os.path.dirname(self.osimg), lnk)
- blk = os.path.basename(lnk)
+ # turboLiveInst ungracefulness
+ # graceful would be replacing this with parsing the output of dumpe2fs
+ if os.path.exists("/dev/shm/osmin.size"):
+ sizefile = "/dev/shm/osmin.size"
+ else:
+ lnk = os.readlink(self.osimg)
+ if lnk[0] != "/":
+ lnk = os.path.join(os.path.dirname(self.osimg), lnk)
+ blk = os.path.basename(lnk)
+
+ if not os.path.exists("/sys/block/%s/size" %(blk,)):
+ log.debug("Unable to determine the actual size of the live image")
+ return 0
- if not os.path.exists("/sys/block/%s/size" %(blk,)):
- log.debug("Unable to determine the actual size of the live image")
- return 0
+ sizefile = "/sys/block/%s/size" %(blk,)
- size = open("/sys/block/%s/size" %(blk,), "r").read()
+ size = open(sizefile, "r").read()
try:
size = int(size)
except ValueError:
diff -Naur anaconda.cvs.20070723/liveinst/liveinst.sh anaconda/liveinst/liveinst.sh
--- anaconda.cvs.20070723/liveinst/liveinst.sh 2007-04-04 13:05:42.000000000 -0500
+++ anaconda/liveinst/liveinst.sh 2007-07-23 02:36:04.000000000 -0500
@@ -4,7 +4,19 @@
#
if [ -z "$LIVE_BLOCK" ]; then
- LIVE_BLOCK="/dev/live-osimg"
+
+ # this may belong elsewhere? /etc/rc.d/init.d/live? seems ungraceful atm
+ if ( losetup /dev/loop118 > /dev/null 2>&1 ); then
+ if [ ! -f /dev/shm/osmin ]; then
+ tar --sparse --directory /dev/shm -xf /dev/loop118
+ fi
+ losetup /dev/loop117 /dev/shm/osmin
+ echo "0 $( blockdev --getsize /dev/loop121 ) snapshot /dev/loop121 /dev/loop117 p 8" | dmsetup create live-osimg-min
+ echo "$(( $(/sbin/dumpe2fs -h /dev/mapper/live-osimg-min 2>/dev/null | grep "^Block size:" | sed -e 's/.*:\s*//') * $(/sbin/dumpe2fs -h /dev/mapper/live-osimg-min 2>/dev/null | grep "^Block count:" | sed -e 's/.*:\s*//') / 512 ))" > /dev/shm/osmin.size
+ LIVE_BLOCK="/dev/mapper/live-osimg-min"
+ else
+ LIVE_BLOCK="/dev/live-osimg"
+ fi
fi
if [ ! -b $LIVE_BLOCK ]; then
