> ... your patches are kind of broken. > Care to resend? Attached. --
From 2e84da8fbb475a75580b8fce4e66eff1ed4f4a1f Mon Sep 17 00:00:00 2001 From: John Reiser <jreiser@xxxxxxxxxxxx> Date: Fri, 26 Aug 2011 13:01:33 -0700 Subject: [PATCH 1/8] inst_simple, inst_dir: make fast case faster This small stuff saves 1.7% per dropped statement during "dracut --profile". Fixing the comment about /lib -> lib64 is REQUIRED! --- dracut-functions | 16 +++++----------- 1 files changed, 5 insertions(+), 11 deletions(-) diff --git a/dracut-functions b/dracut-functions index c28766e..f6ffa1d 100755 --- a/dracut-functions +++ b/dracut-functions @@ -266,19 +266,13 @@ check_vol_slaves() { } # Install a directory, keeping symlinks as on the original system. -# Example: if /lib64 points to /lib on the host, "inst_dir /lib/file" +# Example: if /lib points to /lib64 on the host, "inst_dir /lib/file" # will create ${initdir}/lib64, ${initdir}/lib64/file, # and a symlink ${initdir}/lib -> lib64. inst_dir() { - local _file="" - local _oldifs="$IFS" - local _part - local _dir="$1" + [[ -e ${initdir}"$1" ]] && return 0 # already there - # fast out - [[ -e ${initdir}$_dir ]] && return 0 - - _part=${_dir%/*} + local _dir="$1" _part=${_dir%/*} _file while [[ "$_part" != "${_part%/*}" ]] && ! [[ -e "${initdir}${_part}" ]]; do _dir="$_part $_dir" _part=${_part%/*} @@ -310,9 +304,9 @@ inst_dir() { # Location of the image dir is assumed to be $initdir # We never overwrite the target if it exists. inst_simple() { - local _src target [[ -f $1 ]] || return 1 - _src=$1 target="${2:-$1}" + + local _src=$1 target="${2:-$1}" if ! [[ -d ${initdir}$target ]]; then [[ -e ${initdir}$target ]] && return 0 [[ -h ${initdir}$target ]] && return 0 -- 1.7.6
From e922399ce2f3d49a0cd306b3caf75479142b19e6 Mon Sep 17 00:00:00 2001 From: John Reiser <jreiser@xxxxxxxxxxxx> Date: Sat, 27 Aug 2011 14:43:49 -0700 Subject: [PATCH 2/8] filter_kernel_modules is a specialized filter_kernel_modules_by_path --- dracut-functions | 31 +++---------------------------- 1 files changed, 3 insertions(+), 28 deletions(-) diff --git a/dracut-functions b/dracut-functions index f6ffa1d..15a75b9 100755 --- a/dracut-functions +++ b/dracut-functions @@ -863,34 +863,9 @@ filter_kernel_modules_by_path () ( done ) -# filter kernel modules to install certain modules that meet specific -# requirements. -# $1 = function to call with module name to filter. -# This function will be passed the full path to the module to test. -# The behaviour of this function can vary depending on whether $hostonly is set. -# If it is, we will only look at modules that are already in memory. -# If it is not, we will look at all kernel modules -# This function returns the full filenames of modules that match $1 -filter_kernel_modules () ( - local _modname _filtercmd - if ! [[ $hostonly ]]; then - _filtercmd='find "$srcmods/kernel/drivers" "$srcmods/extra"' - _filtercmd+=' "$srcmods/weak-updates" -name "*.ko" -o -name "*.ko.gz"' - _filtercmd+=' 2>/dev/null' - else - _filtercmd='cut -d " " -f 1 </proc/modules|xargs modinfo -F filename ' - _filtercmd+='-k $kernel 2>/dev/null' - fi - for _modname in $(eval $_filtercmd); do - case $_modname in - *.ko) "$1" "$_modname" && echo "$_modname";; - *.ko.gz) gzip -dc "$_modname" > $initdir/$$.ko - $1 $initdir/$$.ko && echo "$_modname" - rm -f $initdir/$$.ko - ;; - esac - done -) +filter_kernel_modules () { + filter_kernel_modules_by_path drivers "$1" +} # install kernel modules along with all their dependencies. instmods() { -- 1.7.6
From 8d8ce7b1e1f149712a85e811c623ba4056ef4818 Mon Sep 17 00:00:00 2001 From: John Reiser <jreiser@xxxxxxxxxxxx> Date: Sun, 28 Aug 2011 13:24:58 -0700 Subject: [PATCH 3/8] install_kmod_with_fw: make fast case faster --- dracut-functions | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dracut-functions b/dracut-functions index 15a75b9..9d941d7 100755 --- a/dracut-functions +++ b/dracut-functions @@ -782,13 +782,14 @@ check_module_dir() { # Install a single kernel module along with any firmware it may require. # $1 = full path to kernel module to install install_kmod_with_fw() { - local _modname=${1##*/} _fwdir _found _fw - _modname=${_modname%.ko*} # no need to go further if the module is already installed [[ -e "${initdir}/lib/modules/$kernel/${1##*/lib/modules/$kernel/}" ]] \ && return 0 inst_simple "$1" "/lib/modules/$kernel/${1##*/lib/modules/$kernel/}" \ || return $? + + local _modname=${1##*/} _fwdir _found _fw + _modname=${_modname%.ko*} for _fw in $(modinfo -k $kernel -F firmware $1 2>/dev/null); do _found='' for _fwdir in $fw_dir; do -- 1.7.6
From 42953a1782333af1899b82b46f9cb5d1fc3b6b7e Mon Sep 17 00:00:00 2001 From: John Reiser <jreiser@xxxxxxxxxxxx> Date: Mon, 29 Aug 2011 13:40:21 -0700 Subject: [PATCH 4/8] instmods: get filenames from stdin if no args; use it Use bash "[[ string =~ pattern ]]" instead of "egrep -q". Replace control-dominated serial fondling for var in $(proc1); do proc2 var; done with data-dominated parallel pipeline proc1 | while read var; do proc2 var; done Together this is a large savings. --- dracut-functions | 57 ++++++++++++++++++++-------- modules.d/40network/module-setup.sh | 18 ++++++-- modules.d/90kernel-modules/module-setup.sh | 11 +++++- modules.d/90multipath/module-setup.sh | 15 +++++-- modules.d/95iscsi/module-setup.sh | 12 ++++- 5 files changed, 83 insertions(+), 30 deletions(-) diff --git a/dracut-functions b/dracut-functions index 9d941d7..5ff1d26 100755 --- a/dracut-functions +++ b/dracut-functions @@ -863,49 +863,63 @@ filter_kernel_modules_by_path () ( esac done ) +find_kernel_modules_by_path () ( + if ! [[ $hostonly ]]; then + find "$srcmods/kernel/$1" "$srcmods/extra" "$srcmods/weak-updates" \ + -name "*.ko" -o -name "*.ko.gz" 2>/dev/null + else + cut -d " " -f 1 </proc/modules \ + | xargs modinfo -F filename -k $kernel 2>/dev/null + fi +) filter_kernel_modules () { filter_kernel_modules_by_path drivers "$1" } +find_kernel_modules () { + find_kernel_modules_by_path drivers +} + # install kernel modules along with all their dependencies. instmods() { [[ $no_kernel = yes ]] && return - local _mod _mpargs _moddirname - local _ret=0 - while (($# > 0)); do - _mod=${1%.ko*} + + function inst1mod() { + local _mod="$1" case $_mod in =*) # This introduces 2 incompatible meanings for =* arguments # to instmods. We need to decide which one to keep. if [[ $_mod = =ata && -f $srcmods/modules.block ]]; then - instmods $_mpargs \ - $(egrep 'ata|ahci' "${srcmods}/modules.block") + ( echo -n "$_mpargs"; egrep 'ata|ahci' "${srcmods}/modules.block" ) \ + | instmods elif [ -f $srcmods/modules.${_mod#=} ]; then - instmods $_mpargs $(cat ${srcmods}/modules.${_mod#=} ) + ( echo -n "$_mpargs"; cat "${srcmods}/modules.${_mod#=}" ) \ + | instmods else - instmods $_mpargs $(find "$srcmods" -path "*/${_mod#=}/*") + ( echo -n "$_mpargs"; find "$srcmods" -path "*/${_mod#=}/*" ) \ + | instmods fi ;; - --*) - _mpargs+=" $_mod";; - i2o_scsi) shift; continue;; # Do not load this diagnostic-only module + --*) _mpargs+="${_mod##*/}"$'\n' ;; # one _mod per line; lops '--' + i2o_scsi) return ;; # Do not load this diagnostic-only module *) _mod=${_mod##*/} + # if we are already installed, skip this module and go on # to the next one. - [[ -f $initdir/$1 ]] && { shift; continue; } + [[ -f $initdir/$1 ]] && return + # If we are building a host-specific initramfs and this # module is not already loaded, move on to the next one. [[ $hostonly ]] && ! grep -qe "\<${_mod//-/_}\>" /proc/modules \ - && ! echo $add_drivers | grep -qe "\<${_mod}\>" && { - shift; continue - } + && ! echo $add_drivers | grep -qe "\<${_mod}\>" \ + && return # We use '-d' option in modprobe only if modules prefix path # differs from default '/'. This allows us to use Dracut with # old version of modprobe which doesn't have '-d' option. - _moddirname=${srcmods%%/lib/modules/*} + local _moddirname=${srcmods%%/lib/modules/*} [[ -n ${_moddirname} ]] && _moddirname="-d ${_moddirname}/" # ok, load the module, all its dependencies, and any firmware @@ -915,6 +929,17 @@ instmods() { ((_ret+=$?)) ;; esac + } + + local _mpargs _ret=0 + if (($# == 0)); then # filenames from stdin + local _mod + while read _mod; do + inst1mod "${_mod%.ko*}" + done + fi + while (($# > 0)); do # filenames as args + inst1mod ${1%.ko*} shift done return $_ret diff --git a/modules.d/40network/module-setup.sh b/modules.d/40network/module-setup.sh index 39366b6..ff525d4 100755 --- a/modules.d/40network/module-setup.sh +++ b/modules.d/40network/module-setup.sh @@ -24,15 +24,23 @@ depends() { installkernel() { # Include wired net drivers, excluding wireless - net_module_test() { + net_module_filter() { local _net_drivers='eth_type_trans|register_virtio_device' local _unwanted_drivers='/(wireless|isdn|uwb)/' - egrep -q $_net_drivers "$1" && \ - egrep -qv 'iw_handler_get_spy' "$1" && \ - [[ ! $1 =~ $_unwanted_drivers ]] + local _fname + while read _fname; do + local _fcont + case "$_fname" in + *.ko) _fcont="$(< $_fname)" ;; + *.ko.gz) _fcont="$(gzip -dc $_fname)" ;; + esac + [[ $_fcont =~ $_net_drivers + && ! $_fcont =~ iw_handler_get_spy|$_unwanted_drivers ]] \ + && echo "$_fname" + done } - instmods $(filter_kernel_modules_by_path drivers/net net_module_test) + find_kernel_modules_by_path drivers/net | net_module_filter | instmods instmods ecb arc4 # bridge modules diff --git a/modules.d/90kernel-modules/module-setup.sh b/modules.d/90kernel-modules/module-setup.sh index 245ec0b..9fc4248 100755 --- a/modules.d/90kernel-modules/module-setup.sh +++ b/modules.d/90kernel-modules/module-setup.sh @@ -9,6 +9,15 @@ installkernel() { egrep -q "$blockfuncs" "$1" } + block_module_filter() { + local _blockfuncs='ahci_init_controller|ata_scsi_ioctl|scsi_add_host|blk_init_queue|register_mtd_blktrans|scsi_esp_register|register_virtio_device' + local _f + while read _f; do case "$_f" in + *.ko) [[ $(< $_f) =~ $_blockfuncs ]] && echo "$_f" ;; + *.ko.gz) [[ $(gzip -dc <$_f) =~ $_blockfuncs ]] && echo "$_f" ;; + esac + done + } hostonly='' instmods sr_mod sd_mod scsi_dh scsi_dh_rdac scsi_dh_emc hostonly='' instmods pcmcia firewire-ohci hostonly='' instmods usb_storage sdhci sdhci-pci @@ -18,7 +27,7 @@ installkernel() { # install unix socket support hostonly='' instmods unix instmods "=drivers/pcmcia" =ide "=drivers/usb/storage" - instmods $(filter_kernel_modules block_module_test) + find_kernel_modules | block_module_filter | instmods # if not on hostonly mode, install all known filesystems, # if the required list is not set via the filesystems variable if ! [[ $hostonly ]]; then diff --git a/modules.d/90multipath/module-setup.sh b/modules.d/90multipath/module-setup.sh index e9a47fc..f68b58d 100755 --- a/modules.d/90multipath/module-setup.sh +++ b/modules.d/90multipath/module-setup.sh @@ -33,13 +33,18 @@ depends() { } installkernel() { - mp_mod_test() { - local mpfuncs='scsi_register_device_handler|dm_dirty_log_type_register|dm_register_path_selector|dm_register_target' - egrep -q "$mpfuncs" "$1" + mp_mod_filter() { + local _mpfuncs='scsi_register_device_handler|dm_dirty_log_type_register|dm_register_path_selector|dm_register_target' + local _f + while read _f; do case "$_f" in + *.ko) [[ $(< $_f) =~ $_mpfuncs ]] && echo "$_f" ;; + *.ko.gz) [[ $(gzip -dc <$_f) =~ $_mpfuncs ]] && echo "$_f" ;; + esac + done } - instmods $(filter_kernel_modules_by_path drivers/scsi mp_mod_test) - instmods $(filter_kernel_modules_by_path drivers/md mp_mod_test) + ( find_kernel_modules_by_path drivers/scsi; + find_kernel_modules_by_path drivers/md ) | mp_mod_filter | instmods } install() { diff --git a/modules.d/95iscsi/module-setup.sh b/modules.d/95iscsi/module-setup.sh index 3db40ea..b7771ab 100755 --- a/modules.d/95iscsi/module-setup.sh +++ b/modules.d/95iscsi/module-setup.sh @@ -42,11 +42,17 @@ depends() { installkernel() { instmods iscsi_tcp iscsi_ibft crc32c - iscsi_module_test() { + iscsi_module_filter() { local _iscsifuncs='iscsi_register_transport' - fgrep -q "$_iscsifuncs" "$1" + local _f + while read _f; do case "$_f" in + *.ko) [[ $(< $_f) =~ $_iscsifuncs ]] && echo "$_f" ;; + *.ko.gz) [[ $(gzip -dc <$_f) =~ $_iscsifuncs ]] && echo "$_f" ;; + esac + done } - instmods $(filter_kernel_modules_by_path drivers/scsi iscsi_module_test) + find_kernel_modules_by_path drivers/scsi \ + | iscsi_module_filter | instmods } install() { -- 1.7.6
From a78986abef62dc421aae51e9f31df61419e46d7d Mon Sep 17 00:00:00 2001 From: John Reiser <jreiser@xxxxxxxxxxxx> Date: Mon, 29 Aug 2011 14:46:25 -0700 Subject: [PATCH 5/8] instmods: sanity for _mpargs --- dracut-functions | 11 +++++++---- 1 files changed, 7 insertions(+), 4 deletions(-) diff --git a/dracut-functions b/dracut-functions index 5ff1d26..dc71f51 100755 --- a/dracut-functions +++ b/dracut-functions @@ -892,17 +892,20 @@ instmods() { # This introduces 2 incompatible meanings for =* arguments # to instmods. We need to decide which one to keep. if [[ $_mod = =ata && -f $srcmods/modules.block ]]; then - ( echo -n "$_mpargs"; egrep 'ata|ahci' "${srcmods}/modules.block" ) \ + ( [[ "$_mpargs" ]] && echo $_mpargs + egrep 'ata|ahci' "${srcmods}/modules.block" ) \ | instmods elif [ -f $srcmods/modules.${_mod#=} ]; then - ( echo -n "$_mpargs"; cat "${srcmods}/modules.${_mod#=}" ) \ + ( [[ "$_mpargs" ]] && echo $_mpargs + cat "${srcmods}/modules.${_mod#=}" ) \ | instmods else - ( echo -n "$_mpargs"; find "$srcmods" -path "*/${_mod#=}/*" ) \ + ( [[ "$_mpargs" ]] && echo $_mpargs + find "$srcmods" -path "*/${_mod#=}/*" ) \ | instmods fi ;; - --*) _mpargs+="${_mod##*/}"$'\n' ;; # one _mod per line; lops '--' + --*) _mpargs+=" $_mod" ;; i2o_scsi) return ;; # Do not load this diagnostic-only module *) _mod=${_mod##*/} -- 1.7.6
From 94d1f80ecae28bfecf38248a28a494b86551e0fa Mon Sep 17 00:00:00 2001 From: John Reiser <jreiser@xxxxxxxxxxxx> Date: Mon, 29 Aug 2011 16:03:35 -0700 Subject: [PATCH 6/8] instmods: factor out egrep of "FATAL: Module .* not found" --- dracut-functions | 34 +++++++++++++++++++--------------- 1 files changed, 19 insertions(+), 15 deletions(-) diff --git a/dracut-functions b/dracut-functions index dc71f51..6278eb4 100755 --- a/dracut-functions +++ b/dracut-functions @@ -820,7 +820,7 @@ install_kmod_with_fw() { for_each_kmod_dep() { local _func=$1 _kmod=$2 _cmd _modpath _options _found=0 shift 2 - modprobe "$@" --ignore-install --show-depends $_kmod 2>"$initdir/modprobe.err" | ( + modprobe "$@" --ignore-install --show-depends $_kmod 2>&$modprobe_stderr | ( while read _cmd _modpath _options; do [[ $_cmd = insmod ]] || continue $_func ${_modpath} || exit $? @@ -829,9 +829,6 @@ for_each_kmod_dep() { [[ $_found -eq 0 ]] && exit 1 exit 0 ) - egrep -v 'FATAL: Module .* not found.' "$initdir/modprobe.err" | derror - rm -f "$initdir/modprobe.err" - return $? } # filter kernel modules to install certain modules that meet specific @@ -934,16 +931,23 @@ instmods() { esac } - local _mpargs _ret=0 - if (($# == 0)); then # filenames from stdin - local _mod - while read _mod; do - inst1mod "${_mod%.ko*}" + function instmods_1() { + local _ret=0 _mod _mpargs + if (($# == 0)); then # filenames from stdin + while read _mod; do + inst1mod "${_mod%.ko*}" + done + fi + while (($# > 0)); do # filenames as arguments + inst1mod ${1%.ko*} + shift done - fi - while (($# > 0)); do # filenames as args - inst1mod ${1%.ko*} - shift - done - return $_ret + return $_ret + } + + # Capture all stderr from modprobe onto a new fd $modprobe_stderr, + # and pipe it into egrep. See REDIRECTION in bash manpage. + ( instmods_1 "$@" ) {modprobe_stderr}>&1 \ + | egrep -v 'FATAL: Module .* not found.' | derror + return $? } -- 1.7.6
From 4645a5d7517c485042c915c51c07fe9e9ca61727 Mon Sep 17 00:00:00 2001 From: John Reiser <jreiser@xxxxxxxxxxxx> Date: Mon, 29 Aug 2011 22:54:59 -0700 Subject: [PATCH 7/8] build initramfs: cpio avoids per-file /bin/cp Replacing most /bin/cp with one cpio reduces execve and increases parallelism. This is a large savings. TODO: Fix permissions on directories (cpio 775 should be inst_dir 755.) --- dracut | 5 +++++ dracut-functions | 20 ++++++++++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/dracut b/dracut index fd36805..d42f258 100755 --- a/dracut +++ b/dracut @@ -500,6 +500,8 @@ if [[ $prefix ]]; then done fi +export cpio_stdin; ( # begin pipe into cpio to avoid per-file 'cp' + if [[ $kernel_only != yes ]]; then for d in bin etc lib "$libdir" sbin tmp usr var usr/bin usr/sbin; do [[ -e "${initdir}${prefix}/$d" ]] && continue @@ -613,6 +615,9 @@ if [[ $kernel_only != yes ]]; then fi fi +# bash sets cpio_stdin to a new fd >= 10. See REDIRECTION in bash manpage. +) {cpio_stdin}>&1 | cpio -pdmu "${initdir}" + if (($maxloglvl >= 5)); then ddebug "Listing sizes of included files:" du -c "$initdir" | sort -n | ddebug diff --git a/dracut-functions b/dracut-functions index 6278eb4..8109699 100755 --- a/dracut-functions +++ b/dracut-functions @@ -306,6 +306,17 @@ inst_dir() { inst_simple() { [[ -f $1 ]] || return 1 + if [[ 1 = $# || $2 = $1 ]]; then + [[ -e ${initdir}$1 ]] && return 0 + + local _hmac="${1%/*}/.${1##*/}.hmac" + [[ -e $_hmac ]] && echo "$_hmac" >&${cpio_stdin} + + # ddebug "Installing $1" + echo "$1" >&${cpio_stdin} + return 0 + fi + local _src=$1 target="${2:-$1}" if ! [[ -d ${initdir}$target ]]; then [[ -e ${initdir}$target ]] && return 0 @@ -313,9 +324,9 @@ inst_simple() { inst_dir "${target%/*}" fi # install checksum files also - if [[ -e "${_src%/*}/.${_src##*/}.hmac" ]]; then - inst_simple "${_src%/*}/.${_src##*/}.hmac" "${target%/*}/.${target##*/}.hmac" - fi + local _hmac="${_src%/*}/.${_src##*/}.hmac" + [[ -e $_hmac ]] && inst_simple "$_hmac" "${target%/*}/.${target##*/}.hmac" + ddebug "Installing $_src" cp -pfL "$_src" "${initdir}$target" } @@ -820,7 +831,7 @@ install_kmod_with_fw() { for_each_kmod_dep() { local _func=$1 _kmod=$2 _cmd _modpath _options _found=0 shift 2 - modprobe "$@" --ignore-install --show-depends $_kmod 2>&$modprobe_stderr | ( + modprobe "$@" --ignore-install --show-depends $_kmod 2>&${modprobe_stderr} | ( while read _cmd _modpath _options; do [[ $_cmd = insmod ]] || continue $_func ${_modpath} || exit $? @@ -947,6 +958,7 @@ instmods() { # Capture all stderr from modprobe onto a new fd $modprobe_stderr, # and pipe it into egrep. See REDIRECTION in bash manpage. + export modprobe_stderr ( instmods_1 "$@" ) {modprobe_stderr}>&1 \ | egrep -v 'FATAL: Module .* not found.' | derror return $? -- 1.7.6
From eeafddf4d891aeb5bffbaffa06a83c5050ee34c9 Mon Sep 17 00:00:00 2001 From: John Reiser <jreiser@xxxxxxxxxxxx> Date: Tue, 30 Aug 2011 15:02:56 -0700 Subject: [PATCH 8/8] Close original fd after bash-generated high redirection. --- dracut | 3 ++- dracut-functions | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dracut b/dracut index d42f258..c46244f 100755 --- a/dracut +++ b/dracut @@ -616,7 +616,8 @@ if [[ $kernel_only != yes ]]; then fi # bash sets cpio_stdin to a new fd >= 10. See REDIRECTION in bash manpage. -) {cpio_stdin}>&1 | cpio -pdmu "${initdir}" +# Pipe that new fd as stdin into cpio, and ignore the old stdout of subshell. +) {cpio_stdin}>&1 1>&- | cpio -pdmu --quiet "${initdir}" if (($maxloglvl >= 5)); then ddebug "Listing sizes of included files:" diff --git a/dracut-functions b/dracut-functions index 8109699..1942bf9 100755 --- a/dracut-functions +++ b/dracut-functions @@ -959,7 +959,7 @@ instmods() { # Capture all stderr from modprobe onto a new fd $modprobe_stderr, # and pipe it into egrep. See REDIRECTION in bash manpage. export modprobe_stderr - ( instmods_1 "$@" ) {modprobe_stderr}>&1 \ + ( instmods_1 "$@" ) {modprobe_stderr}>&1 1>&- \ | egrep -v 'FATAL: Module .* not found.' | derror return $? } -- 1.7.6