[OS-BUILD PATCHv7 4/9] redhat/configs/process_configs.sh: Parallelize execution

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



From: Prarit Bhargava <prarit@xxxxxxxxxx>

redhat/configs/process_configs.sh: Parallelize execution

Parallelize the execution of each config processing.

The '+' adds job processing to 'non-make' commands [1].

[1] https://www.gnu.org/software/make/manual/html_node/Error-Messages.html

Signed-off-by: Prarit Bhargava <prarit@xxxxxxxxxx>

diff --git a/redhat/Makefile b/redhat/Makefile
index blahblah..blahblah 100644
--- a/redhat/Makefile
+++ b/redhat/Makefile
@@ -50,6 +50,14 @@ KABIDW := $(REDHAT)/kabi-dwarf
 
 include Makefile.cross
 
+RHJOBS?=$(shell j=$$(echo $(MAKEFLAGS) | grep -Eo "\ -j[0-9]*" | xargs ); \
+	if [ -z "$${j}" ]; then \
+		echo "1"; \
+	else \
+		j=$$(echo "$${j}" | tr -d "\-j"); \
+		[ -z "$${j}" ] && nproc --all || echo $${j}; \
+	fi)
+
 default: dist-help
 
 dist-python-check:
@@ -121,12 +129,12 @@ dist-kabi-dw-check: dist-kabi
 	@rm -rf $(KABIDW)/base/$(CURARCH).tmp
 
 dist-configs-commit: dist-configs-prep
-	@cd $(REDHAT)/configs; VERSION=$(KVERSION) ./generate_all_configs.sh "$(FLAVOR)" 1; \
-	./process_configs.sh -z $(PACKAGE_NAME) $(KVERSION) "" $(FLAVOR)
+	+@cd $(REDHAT)/configs; VERSION=$(KVERSION) ./generate_all_configs.sh "$(FLAVOR)" 1; \
+	./process_configs.sh -z "$(PACKAGE_NAME)" "$(KVERSION)" "" "$(FLAVOR)" "$(RHJOBS)"
 
 dist-configs: dist-configs-prep
-	@cd $(REDHAT)/configs; VERSION=$(KVERSION) ./generate_all_configs.sh "$(FLAVOR)" 1; \
-	./process_configs.sh $(PROCESS_CONFIGS_OPTS) $(PACKAGE_NAME) $(KVERSION)
+	+@cd $(REDHAT)/configs; VERSION=$(KVERSION) ./generate_all_configs.sh "$(FLAVOR)" 1; \
+	./process_configs.sh $(PROCESS_CONFIGS_OPTS) "$(PACKAGE_NAME)" "$(KVERSION)" "" "" "$(RHJOBS)"
 
 dist-fedora-configs: FLAVOR = fedora
 dist-fedora-configs: dist-configs
@@ -137,7 +145,7 @@ dist-rhel-configs: dist-configs
 rh-configs: dist-rhel-configs
 
 dist-configs-check: dist-configs-prep
-	cd $(REDHAT)/configs; ./process_configs.sh $(PROCESS_CONFIGS_CHECK_OPTS) $(PACKAGE_NAME)
+	+cd $(REDHAT)/configs; ./process_configs.sh $(PROCESS_CONFIGS_CHECK_OPTS) $(PACKAGE_NAME)
 
 dist-configs-prep: dist-clean-configs dist-buildreq-check
 	cd $(REDHAT)/configs; ./build_configs.sh "partial" "$(ARCH_MACH)" "snip"
diff --git a/redhat/configs/process_configs.sh b/redhat/configs/process_configs.sh
index blahblah..blahblah 100755
--- a/redhat/configs/process_configs.sh
+++ b/redhat/configs/process_configs.sh
@@ -105,8 +105,6 @@ checkoptions()
 		! $checkoptions_error && return
 
 		sed -i "1s/^/Error: Mismatches found in configuration files for ${variant}\n/" .mismatches${count}
-		cat .mismatches${count}
-		RETURNCODE=1
 		[ "$CONTINUEONERROR" ] || exit 1
 	else
 		rm -f .mismatches${count}
@@ -255,8 +253,8 @@ function process_config()
 	grep -E 'CONFIG_' .listnewconfig${count} > .newoptions${count}
 	if test -n "$NEWOPTIONS" && test -s .newoptions${count}
 	then
-		echo "Found unset config items in ${variant}, please set them to an appropriate value"
-		cat .newoptions${count}
+		echo "Found unset config items in ${variant}, please set them to an appropriate value" >> .errors${count}
+		cat .newoptions${count} >> .errors${count}
 		rm .newoptions${count}
 		RETURNCODE=1
 		[ "$CONTINUEONERROR" ] || exit 1
@@ -266,10 +264,9 @@ function process_config()
 	grep -E 'config.*warning' .listnewconfig${count} > .warnings${count}
 	if test -n "$CHECKWARNINGS" && test -s .warnings${count}
 	then
-		echo "Found misconfigured config items in ${variant}, please set them to an appropriate value"
-		cat .warnings${count}
+		echo "Found misconfigured config items in ${variant}, please set them to an appropriate value" >> .errors${count}
+		cat .warnings${count} >> .errors${count}
 		rm .warnings${count}
-		RETURNCODE=1
 		[ "$CONTINUEONERROR" ] || exit 1
 	fi
 	rm .warnings${count}
@@ -302,13 +299,31 @@ function process_configs()
 	count=0
 	for cfg in "$SCRIPT_DIR/${PACKAGE_NAME}${KVERREL}${SUBARCH}"*.config
 	do
-		process_config "$cfg" "$count"
+		process_config "$cfg" "$count" &
+		waitpids[${count}]=$!
 		((count++))
+		while [ "$(jobs | grep Running | wc -l)" -ge $RHJOBS ]; do :; done
+	done
+	for pid in ${waitpids[*]}; do
+		wait ${pid}
 	done
+
 	rm "$SCRIPT_DIR"/*.config*.old
+
+	if ls .errors* 1> /dev/null 2>&1; then
+		RETURNCODE=1
+		cat .errors*
+		rm .errors* -f
+	fi
+	if ls .mismatches* 1> /dev/null 2>&1; then
+		RETURNCODE=1
+		cat .mismatches*
+		rm .mismatches* -f
+	fi
+
 	popd > /dev/null
 
-	echo "Processed config files are in $SCRIPT_DIR"
+	[ $RETURNCODE -eq 0 ] && echo "Processed config files are in $SCRIPT_DIR"
 }
 
 CHECKOPTIONS=""
@@ -369,6 +384,7 @@ PACKAGE_NAME="${1:-kernel}" # defines the package name used
 KVERREL="$(test -n "$2" && echo "-$2" || echo "")"
 SUBARCH="$(test -n "$3" && echo "-$3" || echo "")"
 FLAVOR="$(test -n "$4" && echo "-$4" || echo "-common")"
+RHJOBS="$(test -n "$5" && echo "$5" || nproc --all)"
 SCRIPT=$(readlink -f "$0")
 SCRIPT_DIR=$(dirname "$SCRIPT")
 

--
https://gitlab.com/cki-project/kernel-ark/-/merge_requests/1635
_______________________________________________
kernel mailing list -- kernel@xxxxxxxxxxxxxxxxxxxxxxx
To unsubscribe send an email to kernel-leave@xxxxxxxxxxxxxxxxxxxxxxx
Fedora Code of Conduct: https://docs.fedoraproject.org/en-US/project/code-of-conduct/
List Guidelines: https://fedoraproject.org/wiki/Mailing_list_guidelines
List Archives: https://lists.fedoraproject.org/archives/list/kernel@xxxxxxxxxxxxxxxxxxxxxxx
Do not reply to spam on the list, report it: https://pagure.io/fedora-infrastructure




[Index of Archives]     [Fedora General Discussion]     [Older Fedora Users Archive]     [Fedora Advisory Board]     [Fedora Security]     [Fedora Devel Java]     [Fedora Legacy]     [Fedora Desktop]     [ATA RAID]     [Fedora Marketing]     [Fedora Mentors]     [Fedora Package Announce]     [Fedora Package Review]     [Fedora Music]     [Fedora Packaging]     [Centos]     [Fedora SELinux]     [Coolkey]     [Yum Users]     [Tux]     [Yosemite News]     [KDE Users]     [Fedora Art]     [Fedora Docs]     [USB]     [Asterisk PBX]

  Powered by Linux