[OS-BUILD PATCH 14/20] kabi: add stablelist helpers

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



From: Prarit Bhargava <prarit@xxxxxxxxxx>

kabi: add stablelist helpers

Add helper scripts to update the kabi stablelist checksums and to diff
symtypes output.

redhat/kabi/diff-kabi computes the current symtypes files and compares
it to the reference symtypes stored in the stablelist.

redhat/kabi/update-kabi computes the current checksums and reference
symtypes files and updates the stablelist contents. It does not stage
any changes.

redhat/kabi/symtypes computes symtypes image, preimage or a diff for
a symtypes node.

Signed-off-by: Čestmír Kalina <ckalina@xxxxxxxxxx>
Signed-off-by: Prarit Bhargava <prarit@xxxxxxxxxx>

diff --git a/redhat/kabi/diff-kabi b/redhat/kabi/diff-kabi
new file mode 100755
index blahblah..blahblah 100755
--- /dev/null
+++ b/redhat/kabi/diff-kabi
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+usage_desc() {
+	sed 's/^\t//' <<EOF
+	Diffs the kernel ABI stablelist symtypes information and current symtypes
+	for architecture ARCH.
+EOF
+}
+
+# Called whenever a new symbol checksum is obtained
+# cb_checksum ARCH SYMBOL CHECKSUM FILE SYMVERSIONS_FILE SYMTYPES_FILE
+cb_checksum() {
+	local arch="$1"
+	local symbol="$2"
+	local file="$4"
+	local symtypes="$6"
+
+	if [ ! -e $REDHAT/kabi/kabi-module/kabi_$arch/.$symbol ]; then
+		echo "Reference file for $symbol on $arch not found."
+		return
+	fi
+
+        $REDHAT/kabi/symtypes diff -s $symbol \
+		$REDHAT/kabi/kabi-module/kabi_$arch/.$symbol $symtypes
+
+	symref=${file:0:-1}symref
+	rm -f $symref || :
+	ln -s $(realpath $REDHAT/kabi/kabi-module/kabi_$arch/.$symbol) $symref
+	make -s ${MAKE_ARGS[@]} $symtypes 2>&1 | grep "warning: $symbol:" || :
+	rm -f $symref || :
+}
+
+# Called whenever the generate command finished successfully
+cb_ready() {
+	:
+}
+
+cd "$(git rev-parse --show-toplevel)"
+REDHAT=${REDHAT:-$(pwd)/redhat/}
+
+# Generate new symbol checksums and symvers files
+. $REDHAT/kabi/symtype-generate
diff --git a/redhat/kabi/symtype-generate b/redhat/kabi/symtype-generate
new file mode 100755
index blahblah..blahblah 100755
--- /dev/null
+++ b/redhat/kabi/symtype-generate
@@ -0,0 +1,678 @@
+#!/usr/bin/env bash
+
+unset MAKEFLAGS
+unset CPP
+unset ARCH
+unset SYMBOL
+
+set -euo pipefail
+
+usage() {
+	sed 's/^\t//' <<EOF
+	Usage: $0 [-h] [-v] [-1] [-2] [-3] [-4] [-5] -a ARCH [-s SYMBOL]... [FILE]...
+EOF
+
+	echo
+	[ "$(type -t usage_desc || :)" == "function" ] && usage_desc
+	echo
+
+	sed 's/^\t//' <<EOF
+	-a ARCH				Supported architectures:
+					x86_64, s390x, aarch64, ppc64le
+
+	-s SYMBOL [-s SYMBOL ...]	Symbol entry to update.
+					Updates the whole stablelist if omitted.
+
+	-h				Prints this message.
+
+	-v				Verbose output.
+
+	== DESCRIPTION
+
+	To calculate current symbol data, the file exporting the symbol on a given
+	arch must first be located. The following methods are attempted in order:
+
+	  1. user-supplied FILEs, if given, (disable using -1)
+	  2. stablelist-supplied FILEs, (disable using -2)
+	  3. cscope, if available, (disable using -3)
+	  4. naive regular expression search for exporting macro (disable
+	     using -4),
+	  5. compute all symvers and symtypes files (disable using -5)
+	  6. compute all symvers and symtypes files following a local kernel
+	     build (disable using -6).
+
+	If a method is successful in updating all of the required symbols,
+	the script early terminates. Methods are sorted by their time complexity
+	as well as generality.
+
+	If you're using the tool directly, please make sure to commit/stash your
+	changes.
+
+	You must also make sure that you have cross compilation toolchain
+	installed when computing non-native checksums. The tool uses
+	  CROSS_COMPILE=/usr/bin/ARCH-linux-gnu-, override
+	  KABI_CROSS_COMPILE_PREFIX=/usr/bin/
+	  KABI_CROSS_COMPILE_SUFFIX=-linux-gnu-
+	as needed.
+
+	== NOTES
+
+	Method 3 can only find symbols exported using one of the following
+	macros:
+
+		ACPI_EXPORT_SYMBOL
+		ACPI_EXPORT_SYMBOL_INIT
+		EXPORT_DATA_SYMBOL
+		EXPORT_DATA_SYMBOL_GPL
+		EXPORT_INDIRECT_CALLABLE
+		EXPORT_PER_CPU_SYMBOL
+		EXPORT_PER_CPU_SYMBOL_GPL
+		EXPORT_STATIC_CALL
+		EXPORT_STATIC_CALL_GPL
+		EXPORT_STATIC_CALL_TRAMP
+		EXPORT_STATIC_CALL_TRAMP_GPL
+		EXPORT_SYMBOL
+		EXPORT_SYMBOL_FOR_TESTS_ONLY
+		EXPORT_SYMBOL_GPL
+		EXPORT_SYMBOL_NS
+		EXPORT_SYMBOL_NS_GPL
+		EXPORT_TRACEPOINT_SYMBOL
+		EXPORT_TRACEPOINT_SYMBOL_GPL
+
+	and the symbol name must not be constructed using preprocessor
+	concatenation.
+EOF
+	exit
+}
+
+# --- decls
+
+# list of temporary files created during execution
+declare -ag TEMP_FILES
+
+# list of makefile targets to invoke
+declare -ag TARGETS
+
+# list of source files to process
+declare -ag SOURCES
+
+# list of symbols to try and generate symbol versions/symtypes for
+declare -ag SYMBOL
+
+# list of symbols that couldn't have been resolved using previous method
+declare -ag SYMFAIL
+
+# current arch
+declare -g CARCH
+
+# dict[arch]=ARCH variable for kernel makefile
+declare -Ag CC_ARCH
+
+# dict[symbol]=checksum
+# dict[symbol]=file_exporting_symbol
+declare -Ag SYMCRC
+declare -Ag SYMFILE
+declare -Ag SYMFILE_T
+declare -Ag SYMFILE_V
+
+# kernel makefile cc var
+declare -g CROSS_COMPILE
+
+# compiler
+declare -g CPP
+
+# verbose level
+declare -g V
+
+# --- default assignments
+
+V=${V:-0}
+CPP=gcc
+KABI_CROSS_COMPILE_PREFIX="${KABI_CROSS_COMPILE_PREFIX:-/usr/bin/}"
+KABI_CROSS_COMPILE_SUFFIX="${KABI_CROSS_COMPILE_SUFFIX:--linux-gnu-}"
+USE_ENTIRE_STABLELIST=1
+MAKE_ARGS=(-k -i -j$(nproc))
+CC_ARCH[s390x]="s390"
+CC_ARCH[x86_64]="x86"
+CC_ARCH[ppc64le]="powerpc"
+CC_ARCH[aarch64]="arm64"
+
+# --- helper fns
+
+echo0() { echo " :: $*"; }
+echo1() { echo "    $*"; }
+echo2() { echo "     - $*"; }
+echo3() { echo "       $*"; }
+err()   { echo -e "ERROR:" "$@" >&2; }
+warn()  { echo -e "WARNING:" "$@" >&2; }
+
+cleanup() {
+	[ -z "${TEMP_FILES[@]:-}" ] && return
+	echo0 "Cleaning up temporary files ..."
+	rm -f "${TEMP_FILES[@]}"
+}
+
+# find files containing symbol export statements of the form MACRO(NAME),
+# where symbol name NAME is the entire symbol name and MACRO is one of:
+# 	ACPI_EXPORT_SYMBOL
+# 	ACPI_EXPORT_SYMBOL_INIT
+# 	EXPORT_DATA_SYMBOL
+# 	EXPORT_DATA_SYMBOL_GPL
+# 	EXPORT_INDIRECT_CALLABLE
+# 	EXPORT_PER_CPU_SYMBOL
+# 	EXPORT_PER_CPU_SYMBOL_GPL
+# 	EXPORT_STATIC_CALL
+# 	EXPORT_STATIC_CALL_GPL
+# 	EXPORT_STATIC_CALL_TRAMP
+# 	EXPORT_STATIC_CALL_TRAMP_GPL
+# 	EXPORT_SYMBOL
+# 	EXPORT_SYMBOL_FOR_TESTS_ONLY
+# 	EXPORT_SYMBOL_GPL
+# 	EXPORT_SYMBOL_NS
+# 	EXPORT_SYMBOL_NS_GPL
+# 	EXPORT_TRACEPOINT_SYMBOL
+#	EXPORT_TRACEPOINT_SYMBOL_GPL
+# export_try_find SYMBOL|REGEX
+export_tryfind() {
+        grep -Prol --exclude-dir=redhat "^(ACPI_)?EXPORT_(DATA_|INDIRECT_CALLABLE|STATIC_CALL|TRACEPOINT_)?(PER_CPU_)?(SYMBOL|_TRAMP)?(_NS)?(_GPL|_INIT)?\($1(,[^)]*)?\);" || :
+}
+
+symbol_tryfind_symversions() {
+	# __crc_SYMBOL = CHECKSUM
+	# SECTIONS { .rodata : ALIGN(4) { __crc_SYMBOL = .; LONG(CHECKSUM); } }
+	grep -R --include="*.symversions" --exclude-dir=redhat -H -Po \
+		"^__crc_\K$1 = .*(?=;)" | tr -d ' ' && return || :
+	grep -R --include "*.symversions" --exclude-dir=redhat -H -Po \
+		"__crc_\K$1[ =].*0x[[:xdigit:]]+" \
+	| sed -e 's/ = .*\(0x[[:xdigit:]]\+\).*/=\1/' 
+
+}
+
+# batch call to export_tryfind, limited to 10 symbols at the time to make sure
+# argv doesn't overflow
+# export_tryfind SYMBOL...
+batch() {
+	local fn=$1
+	shift 1
+	local o=1
+	local d=10
+	while [ $o -le $# ]; do
+		[ $o -gt $# ] && d=$((o-$#))
+		# pass an array of symbols S1...Sn as a regex (S1|...|Sn)
+		$fn $(
+			echo ${@:$o:$d} \
+			| sed -e 's/ /|/g' -e 's/^/(/' -e 's/$/)/'
+		)
+		o=$((o+d))
+	done | sort | uniq
+}
+
+src_to_symversions() {
+	sed 's/\.[cS]$/.symversions/'
+}
+
+src_to_symtypes() {
+	sed 's/\.[cS]$/.symtypes/'
+}
+
+dump_failed_symbols() {
+	echo1 "Couldn't find the following symbols using this method:" >&3
+	for sym in ${SYMFAIL[@]}; do
+		echo2 $sym >&3
+	done
+}
+
+prepare_stage() {
+	# SYMBOL lists symbols to be searched
+	# SYMFAIL lists symbols that failed in the previous run
+	SYMBOL=(${SYMFAIL[@]})
+	SYMFAIL=()
+	TARGETS=()
+	FILE=()
+}
+
+process_Module_symvers() {
+	[ ! -e Module.symvers ] && return 1
+
+	# read checksums into SYMCRC
+	local IFS=$'\n'
+	for sym in $(comm -12 <(cut -f2 Module.symvers | sort) \
+	                      <(echo ${SYMBOL[@]} | tr ' ' '\n' | sort)); do
+		symcrc=$(grep -Po "^0x[[:xdigit:]]+(?=\t$sym\t)" Module.symvers)
+		file=$(grep -El -m 1 -r --include="*.symtypes" \
+			--exclude-dir=redhat "^[a-zA-Z#]{,2}$sym " | head -n1)
+		[ -z "$file" ] && continue
+		SYMCRC[$sym]=$symcrc
+		SYMFILE_T[$sym]=$file
+		SYMFILE_V[$sym]=Module.symvers
+		[ -e "${file/.symtypes/.c}" ] && file="${file/.symtypes/.c}"
+		[ -e "${file/.symtypes/.S}" ] && file="${file/.symtypes/.S}"
+		SYMFILE[$sym]=$file
+		echo3 "Found $sym in $file (crc: $symcrc)," \
+			"V: ${SYMFILE_V[$sym]}," \
+			"T: ${SYMFILE_T[$sym]}," \
+			"F: ${SYMFILE[$sym]}." >&3
+	done
+
+	return 0
+}
+
+process_symversions() {
+	# read checksums into SYMCRC and update stablelist, output format:
+	# filename.symversions:__crc_symbolname = 0x0000..
+	for match in $(batch symbol_tryfind_symversions "$@"); do
+		file=${match%:*}
+		symcrc=${match#*:}
+		sym=${symcrc%%=*}
+		SYMCRC[$sym]=${symcrc##*=}
+		SYMFILE_V[$sym]=$file
+		SYMFILE_T[$sym]=${file/.symversions/.symtypes}
+		[ -e ${file/.symversions/.c} ] && file=${file/.symversions/.c}
+		[ -e ${file/.symversions/.S} ] && file=${file/.symversions/.S}
+		SYMFILE[$sym]=$file
+		echo3 "Found $sym in $file (crc: $symcrc)," \
+			"V: ${SYMFILE_V[$sym]}," \
+			"T: ${SYMFILE_T[$sym]}," \
+			"F: ${SYMFILE[$sym]}." >&3
+	done
+}
+
+# generate MAKE_TARGET...
+generate() {
+	if [ $# -eq 0 ]; then
+		echo "No targets given to generate."
+		SYMFAIL=(${SYMBOL[@]})
+		return 1
+	fi
+
+	echo1 "Building symtype and symversion files ..."
+	for file in $@; do
+		echo2 "$file" >&3
+	done
+
+	# batch build at most 100 targets (symvers, symtypes files)
+	while [ $# -gt 0 ]; do
+		if [ $# -gt 100 ]; then
+			make ${MAKE_ARGS[@]} ${@:1:100} >&3 2>&3 || :
+			shift 100
+		else
+			make ${MAKE_ARGS[@]} ${@:1:$#} >&3 2>&3 || :
+			shift $#
+		fi
+	done
+
+	if ! process_Module_symvers; then
+		process_symversions "${SYMBOL[@]}"
+	fi
+
+	echo1 "Looking for eligible symbol checksum updates ..." >&3
+	for sym in ${SYMBOL[@]}; do
+		if [ -z "${SYMCRC[$sym]:-}" ]; then
+			echo3 "Couldn't find crc for $sym" >&3
+			[[ ! " ${SYMFAIL[*]} " =~ " $sym " ]] && SYMFAIL+=($sym)
+			continue
+		fi
+		if [ ! -e "${SYMFILE[$sym]:-}" ]; then
+			echo3 "Couldn't find source file for $sym" >&3
+			continue
+		fi
+		if [ ! -e "${SYMFILE_V[$sym]:-}" ]; then
+			echo3 "Couldn't find version file for $sym" >&3
+			continue
+		fi
+		if [ ! -e "${SYMFILE_T[$sym]:-}" ]; then
+			echo3 "Couldn't find symtypes file for $sym" >&3
+			continue
+		fi
+		cb_checksum $CARCH $sym ${SYMCRC[$sym]:-} ${SYMFILE[$sym]} \
+			${SYMFILE_V[$sym]} ${SYMFILE_T[$sym]}
+	done
+
+	if [ -z "${SYMFAIL:-}" ]; then
+		cb_ready
+		return 0
+	fi
+
+	return 1
+}
+
+# process argv
+
+OPTIND=1
+
+while getopts "TC12345hva:f:s:" opt; do
+	case "$opt" in
+	T)
+		CONFIG_NO_TEST=y
+		;;
+	C)
+		CONFIG_NO_CLEAN=y
+		;;
+	1)
+		CONFIG_NO_METH1=y
+		;;
+	2)
+		CONFIG_NO_METH2=y
+		;;
+	3)
+		CONFIG_NO_METH3=y
+		;;
+	4)
+		CONFIG_NO_METH4=y
+		;;
+	5)
+		CONFIG_NO_METH5=y
+		;;
+	6)
+		CONFIG_NO_METH6=y
+		;;
+	h)
+		usage
+		;;
+	v)
+		V=1
+		;;
+	s)
+		USE_ENTIRE_STABLELIST=0
+		[ -z "${CARCH:-}" ] && usage
+		if [ "$OPTARG" = "-" ]; then
+			SYMBOL+=($(cat))
+			continue
+		fi
+		SYMBOL+=($OPTARG)
+		;;
+	a)
+		CARCH=$OPTARG
+		;;
+	esac
+done
+shift $((OPTIND-1))
+
+CARCH=${CARCH:-$(uname -m)}
+SOURCES=("${@:-}")
+
+if [ "$CARCH" = "$(uname -m)" ]; then
+	if ! command -v "$CPP" &> /dev/null; then
+		echo "ERROR: native gcc not found."
+		exit 1
+	fi
+else
+	if [ -z "${CROSS_COMPILE:-}" ]; then
+		CROSS_COMPILE="${KABI_CROSS_COMPILE_PREFIX}"
+		CROSS_COMPILE+="$CARCH"
+		CROSS_COMPILE+="${KABI_CROSS_COMPILE_SUFFIX}"
+	fi
+
+	if ! [ -e "$CROSS_COMPILE$CPP" ]; then
+		echo "ERROR: $arch $CPP not found ($CROSS_COMPILE$CPP)"
+		exit 1
+	fi
+
+	MAKE_ARGS+=(ARCH=${CC_ARCH[$CARCH]} CROSS_COMPILE=$CROSS_COMPILE)
+fi
+
+# Set up is_verbose fd
+if [ ${V:-0} -gt 0 ]; then
+	exec 3>&1
+	export PS4='$LINENO: '
+	set -x
+else
+	exec 3> /dev/null
+fi
+
+export BASH_XTRACEFD="3"
+
+if [ -z "$CARCH" ]; then
+	err "No architecture specified."
+	usage
+fi
+
+if [ ! -d "$REDHAT/kabi/kabi-module/kabi_$CARCH" ]; then
+	err "Architecture $CARCH not supported."
+	exit 1
+fi
+
+trap cleanup EXIT
+
+# --- prep
+
+cd "$(git rev-parse --show-toplevel)"
+
+REDHAT=${REDHAT:-$(pwd)/redhat/}
+
+if [ $USE_ENTIRE_STABLELIST -eq 1 ]; then
+	SYMBOL=($(find $REDHAT/kabi/kabi-module/kabi_$CARCH -type f \
+		-not -name ".*" -exec basename {} \; | sort | uniq))
+	if [ -z "${SYMBOL[*]}" ]; then
+		err "No symbols found on stablelist. Nothing to do."
+		exit
+	fi
+elif [ -z "${SYMBOL[*]}" ]; then
+	err "No symbols given. Nothing to do."
+	exit
+fi
+
+echo0 "The following symbol entries will be updated:"
+
+for symbol in ${SYMBOL[@]}; do
+	if find $REDHAT/kabi/kabi-module/${CARCH/#/kabi_} -name $symbol \
+			-exec false {} + &> /dev/null; then
+		echo1 "$symbol (not found on ${CARCH} stablelist)"
+	else
+		echo1 "$symbol"
+	fi
+done
+
+if [ -z "${CONFIG_NO_CLEAN+x}" ]; then
+	make -j$(nproc) V=$V mrproper >&3 2>&3
+fi
+
+if [ ! -e $REDHAT/configs/kernel*$CARCH.config ]; then
+	echo0 "Generating config files ..."
+	make dist-configs V=$V >&3
+fi
+
+cp $REDHAT/configs/kernel*$(uname -m).config .config
+
+echo0 "Building scripts (genksyms) ..."
+
+make -j$(nproc) V=$V scripts >&3 2>&3 # -> genksyms, requires native cc
+
+cp $REDHAT/configs/kernel*$CARCH.config .config
+
+if [ -z "${CONFIG_NO_TEST+x}" ]; then
+	TEST_FILE=$({ grep -rl EXPORT_SYMBOL net/core net/ . || : ; } | head -n1)
+	TEST_SYMVERSIONS="$(printf "$TEST_FILE" | src_to_symversions)"
+	TEST_SYMTYPES="$(printf "$TEST_FILE" | src_to_symtypes)"
+	echo1 "Checking that genksyms works as expected on a test file" \
+	      "$TEST_FILE, expecting $TEST_SYMVERSIONS, $TEST_SYMTYPES ..." >&3
+	make V=$V ${MAKE_ARGS[@]} $TEST_SYMVERSIONS $TEST_SYMTYPES >&3 2>&3
+	if [ ! -e $TEST_SYMVERSIONS -o ! -e $TEST_SYMTYPES ]; then
+		warn "An attempt to test genksyms failed. Please re-run with" \
+			"-v and inspect the output and git diff to make sure" \
+			"that the script works correctly."
+	fi
+fi
+
+# ---
+
+queue_target() {
+	for f in "$@"; do
+		symv=($(printf "$f" | src_to_symversions))
+		symt=($(printf "$f" | src_to_symtypes))
+		
+		# ensure it's not queued already
+		if [[ ! " ${FILE[*]} " =~ " $symv " ]]; then
+			FILE+=($symv)
+			echo1 "$f: queued targets: $symv" >&3
+		fi
+
+		if [[ ! " ${FILE[*]} " =~ " $symt " ]]; then
+			FILE+=($symt)
+			echo1 "$f: queued targets: $symt" >&3
+		fi
+	done
+}
+
+targets_from_srclist() {
+	for src in "${@:-}"; do
+		if ! [ -e "$src" ]; then
+			err "File \`$src' not found!"
+			exit 1
+		fi
+		queue_target "$src"
+	done
+}
+
+targets_from_symlist() {
+	for sym in ${@:-}; do
+		if ! [ -e $REDHAT/kabi/kabi-module/kabi_$CARCH/$sym ]; then
+			echo1 "$sym not found in $CARCH stablelist, skipping" \
+			      "(missing: $REDHAT/kabi/kabi-module/kabi_$CARCH/$sym)" >&3
+			[[ ! " ${SYMFAIL[*]} " =~ " $sym " ]] && SYMFAIL+=($sym)
+			continue
+		fi
+		src="$(grep -Po '^#P:\K.*' $REDHAT/kabi/kabi-module/kabi_$CARCH/$sym || :)"
+		if [ -z "$src" ]; then
+			echo1 "$sym does not reference source file, skipping" \
+			      "(missing: $REDHAT/kabi/kabi-module/kabi_$CARCH/$sym)" >&3
+			[[ ! " ${SYMFAIL[*]} " =~ " $sym " ]] && SYMFAIL+=($sym)
+			continue
+		fi
+		if ! [ -e "$src" ]; then
+			echo1 "$sym: source $src got moved or removed, skipping" >&3
+			[[ ! " ${SYMFAIL[*]} " =~ " $sym " ]] && SYMFAIL+=($sym)
+			continue
+		fi
+		queue_target "$src"
+	done
+}
+
+targets_cscope() {
+	command -v cscope >&3 || return
+	for sym in "${@:-}"; do
+		queue_target $(cscope -k -d -L1$sym | cut -f1 -d' ' || :)
+	done
+}
+
+targets_from_symlist_any() {
+	for sym in ${@:-}; do
+		fil=($(find $REDHAT/kabi/kabi-module/ -type f -name "$sym"))
+		if [ ${#fil[@]} -eq 0 ]; then
+			echo1 "$sym not found in any stablelist, skipping" \
+			      "(missing: $REDHAT/kabi/kabi-module/*/$sym)" >&3
+			[[ ! " ${SYMFAIL[*]} " =~ " $sym " ]] && SYMFAIL+=($sym)
+			continue
+		fi
+		for f in ${fil[@]}; do
+			queue_target $(grep -Po '^#P:\K.*' $f | sort | uniq || :)
+		done
+	done
+}
+
+targets_naive() {
+	# symbol got moved, naive greedy search for export statement
+	# can produce any non-negative number of results:
+	#	#res > 1 : arch-specific code may yield multiple files in arch,
+	#                  leaving it to kbuild to fail instead of trying to
+	#                  mask them
+	#	#res = 1 : ok, provided this is not a false positive (we can
+	#                  tell that when we generate checksums)
+	#       #res = 0 : this may occur when the export symbol call/symbol
+	#                  name is composed using preprocessor concatenation;
+	#                  in this case we are forced to generate all checksums
+	for src in $(batch export_tryfind "$@"); do
+		queue_target "$src"
+	done
+}
+
+targets_dry_run() {
+	queue_target $({ make ${MAKE_ARGS[@]} --dry-run 2>&3 || : ; } \
+	| grep -E '(gcc|as)' \
+	| grep -Po " \K[a-zA-Z0-9][^ ,.]*[^/ ,.]\.[cS]" \
+	| sort \
+	| uniq \
+	| sed 's/^\(.*\)\.[cS]$/\1.symtypes\n\1.symversions/g')
+}
+
+targets_compile() {
+	declare -A wrappers
+	local template=$(mktemp -t -u "kabi-wrapper-XXXXXXXXX")
+	local list=$(mktemp -u "kabi-list-XXXXXXXXX")
+	TEMP_FILES=("${TEMP_FILES[@]}" "$list")
+	for wrapper in $(grep -Po '= \$\(CROSS_COMPILE\)\K.*' Makefile); do
+		wrappers[$wrapper]=$template-$wrapper
+		TEMP_FILES=("${TEMP_FILES[@]}" "${wrappers[$wrapper]}")
+		bin=${CROSS_COMPILE:-}$wrapper list=$list envsubst '$bin $list' \
+			> "${wrappers[$wrapper]}" <<-'EOF'
+		#!/usr/bin/env bash
+		echo "$*" | grep -Po " \K[a-zA-Z0-9_/-]+\.[cS]" | tr ' ' '\n' >> $list
+		$bin "$@"
+		EOF
+		chmod +x "${wrappers[$wrapper]}"
+	done
+	make ${MAKE_ARGS[@]} CROSS_COMPILE="$template-" >&3 2>&3 || :
+	# no need to build symvers, using Module.symvers instead
+	queue_target $(cat $list | src_to_symtypes | sort | uniq)
+}
+
+SYMFAIL=(${SYMBOL[@]})
+
+if [ -z "${CONFIG_NO_METH1+x}" -a $# -gt 0 ]; then
+	prepare_stage
+	echo0 "Updating stablelist using user-supplied files ..."
+	targets_from_srclist "${SOURCES[@]}"
+	generate "${FILE[@]}" && exit 0 || :
+	dump_failed_symbols
+fi
+
+if [ -z "${CONFIG_NO_METH2+x}" ]; then
+	prepare_stage
+	echo0 "Updating stablelist using stablelist-provided files (arch specific) ..."
+	targets_from_symlist "${SYMBOL[@]}"
+	generate "${FILE[@]}" && exit 0 || :
+	dump_failed_symbols
+
+	prepare_stage
+	echo0 "Updating stablelist using stablelist-provided files (any archs) ..."
+	targets_from_symlist_any "${SYMBOL[@]}"
+	generate "${FILE[@]}" && exit 0 || :
+	dump_failed_symbols
+fi
+
+if [ -z "${CONFIG_NO_METH3+x}" ]; then
+	prepare_stage
+	echo0 "Updating stablelist using cscope-provided files ..."
+	targets_cscope "${SYMBOL[@]}"
+	generate "${FILE[@]}" && exit 0 || :
+	dump_failed_symbols
+fi
+
+if [ -z "${CONFIG_NO_METH4+x}" ]; then
+	prepare_stage
+	echo0 "Updating stablelist using naive method ..."
+	targets_naive "${SYMBOL[@]}"
+	generate "${FILE[@]}" && exit 0 || :
+	dump_failed_symbols
+fi
+
+if [ -z "${CONFIG_NO_METH5+x}" ]; then
+	prepare_stage
+	echo0 "Updating stablelist greedy method (this might take some time) ..."
+	targets_dry_run
+	generate "${FILE[@]}" && exit 0 || :
+	dump_failed_symbols
+fi
+
+if [ -z "${CONFIG_NO_METH6+x}" ]; then
+	prepare_stage
+	echo0 "Updating stablelist by compiling the kernel (this might take some time) ..."
+	targets_compile
+	generate "${FILE[@]}" && exit 0
+fi
+
+err "could not update all of the symbol checksums required:"
+for sym in ${SYMFAIL[@]}; do
+	printf "\t%s\n" "$sym"
+done
+exit 1
diff --git a/redhat/kabi/symtypes b/redhat/kabi/symtypes
new file mode 100755
index blahblah..blahblah 100755
--- /dev/null
+++ b/redhat/kabi/symtypes
@@ -0,0 +1,238 @@
+#!/usr/bin/env python3
+
+import os, sys, argparse, re, difflib, json
+
+def jsonKeys2int(x):
+    try:
+        if isinstance(x, dict):
+            return {int(k):v for k,v in x.items()}
+        return x
+    except ValueError:
+        return x
+
+def symtypes_parse(path, data = None):
+    if not data:
+        data = {
+                'children': { 0 : []},
+                'parents': { 0 : []},
+                'strtab': ["(root)"],
+                'index' : {},
+                'file' : { }
+        }
+
+    bpath = os.path.basename(path)
+    data["file"] = {}
+    data["file"][bpath] = { 0 : "" }
+
+    with open(path, 'r') as fp_ref:
+        for line in fp_ref.readlines():
+            lsplit = line.split(' ')
+            root = lsplit.pop(0)
+            children = list(filter(lambda x: len(x) > 2 and (x[1] == '#' or x == "UNKNOWN"), lsplit))
+            if root in data["index"]:
+                idx = data["index"][root]
+                if idx in data["children"] and len(data["children"][idx]) > 0:
+                    continue
+            index = data_add(data, root, 0, bpath, line)
+            for child in children:
+                child_index = data_add(data, child, index, "", "")
+    return data
+
+def data_add(data, ident, parent, bpath, line):
+    index = data['strtab'].index(ident) if ident in data['strtab'] else -1
+    if index == -1:
+        index = len(data['strtab'])
+        data['strtab'].append(ident)
+        data['index'][ident] = index
+    data['children'][parent].append(index)
+    if index not in data['children']:
+        data['children'][index] = []
+    if index in data['parents']:
+        if parent not in data['parents'][index]:
+            data['parents'][index].append(parent)
+    else:
+        data['parents'][index] = [parent]
+    if bpath and line:
+        data['file'][bpath][index] = line
+    return index
+
+#def symtypes_dfs(data_a, source, sink, trace, inverse = False):
+# print(' > '.join(list(map(lambda i: data_a['strtab'][i], path + [e]))))
+def symtypes_dfs(data, start, inverse=False, full=False):
+    start_i = data["index"][start]
+    stack = [(start_i,[start_i])]
+    visited = set()
+    paths = []
+    while stack:
+        (node, path) = stack.pop()
+        if full and node in path[:-1]:
+            continue
+        if not full:
+            if node in visited:
+                continue
+            visited.add(node)
+        paths.append(path)
+        if not inverse:
+            for child in reversed(data["children"][node]):
+                stack.append((child, path + [child]))
+        else:
+            for child in reversed(data["parents"][node]):
+                if child == 0:
+                    continue
+                stack.append((child, path + [child]))
+
+    return visited, paths
+
+def st_open(path):
+    if not path:
+        raise ValueError("Blank blank.")
+    if not os.path.exists(path):
+        raise OSError(f"Path {path} does not exist.")
+    with open(path, "r") as f:
+        try:
+            return json.load(f, object_hook=jsonKeys2int)
+        except ValueError:
+            pass
+    return symtypes_parse(path)
+
+def st_write(path, data):
+    with open(path, "w+") as f:
+        if "file" in data:
+            del data["file"]
+        json.dump(data, f)
+
+def index(symtype, output):
+    data = st_open(symtype)
+    if output:
+        st_write(output, data)
+    return data
+
+def st_print(node):
+    if node[1] != '#':
+        return node
+    if node[0] == 's':
+        return "struct " + node[2:]
+    if node[0] == 't':
+        return "typedef " + node[2:]
+    if node[0] == 'E':
+        return "enum const " + node[2:]
+    if node[0] == 'e':
+        return "enum " + node[2:]
+    if node[0] == 'u':
+        return "union " + node[2:]
+    return node
+
+def im(file, dump_list, dump_path, dump_tree, start, inverse, silent):
+    data = index(file, None)
+
+    if start not in data["index"]:
+        if not silent:
+            print(f"Node {start} not found in file {file}. Exitting.")
+        sys.exit(1)
+
+    nodes, paths = symtypes_dfs(data, start, inverse)
+
+    if dump_list:
+        for node in map(lambda i: data['strtab'][i], nodes):
+            print(f"{st_print(node)} (symtype node: {node})")
+
+    if not dump_path and not dump_tree:
+        return
+
+    for path in paths:
+        if dump_tree:
+            print((len(path)-1)*"  " + " - " + f"{st_print(data['strtab'][path[-1]])} (symtype node: {data['strtab'][path[-1]]})");
+            continue
+        if dump_path:
+            print(list(map(lambda i: data["strtab"][i], path)))
+
+def diff(ref, new, start):
+    data_ref = index(ref, None)
+    data_new = index(new, None)
+
+    nodes_ref, _ = symtypes_dfs(data_ref, start)
+    nodes_new, _ = symtypes_dfs(data_new, start)
+
+    nodes_ref_lbl = set(map(lambda i: data_ref['strtab'][i], nodes_ref))
+    nodes_new_lbl = set(map(lambda i: data_new['strtab'][i], nodes_new))
+    nodes_all = nodes_ref_lbl | nodes_new_lbl
+    nodes_13 = nodes_all - nodes_ref_lbl
+    nodes_23 = nodes_all - nodes_new_lbl
+
+    if nodes_23:
+        print("The following nodes were encountered only in reference symtypes:")
+        print("\t" + "\n\t".join(nodes_23))
+
+    if nodes_13:
+        print("The following nodes were encountered only in new symtypes:")
+        print("\t" + "\n\t".join(nodes_13))
+
+    bpath_a = os.path.basename(ref)
+    bpath_b = os.path.basename(new)
+    for node in nodes_all - (nodes_13 | nodes_23):
+        idx_a = data_ref['index'][node]
+        idx_b = data_new['index'][node]
+        r = set(map(lambda i: data_ref['strtab'][i], data_ref['children'][idx_a]))
+        n = set(map(lambda i: data_new['strtab'][i], data_new['children'][idx_b]))
+
+        if r == n:
+            continue
+
+        i = ["\t"+data_ref['file'][bpath_a][idx_a]], \
+            ["\t"+data_new['file'][bpath_b][idx_b]]
+
+        if i[0] != i[1]:
+            print(f"Possible breakage detected for {st_print(node)} (symtype node: {node}) ...")
+            if len(n) == 1 and "UNKNOWN" in n:
+                print("\treplaced by UNKNOWN. Please inspect changes to #include directives")
+            if len(r) == 1 and "UNKNOWN" in r:
+                print("\tUNKNOWN got replaced. Please inspect changes to #include directives")
+            diff = difflib.ndiff(i[0], i[1])
+            print(''.join(diff), end="")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+
+    subparsers = parser.add_subparsers(help='Modes of operation.',
+            dest="mode")
+    parser_index = subparsers.add_parser('index',
+            help='Calculate symtypes index.')
+    parser_image = subparsers.add_parser('image',
+            help='Show type/symbol dependencies.')
+    parser_preimage = subparsers.add_parser('preimage',
+            help='Show type/symbol preimage.')
+    parser_df = subparsers.add_parser('diff',
+            help='Calculate simple symtype diff.')
+
+    parser_index.add_argument('-o', '--output', type=str, required=True,
+            help='Output index file.')
+    parser_index.add_argument('symtype', type=str)
+
+    for p in [ parser_image, parser_preimage ]:
+        p.add_argument('-i', '--index', action='store_true',
+                help="Input is an index file.")
+        p.add_argument('-S', '--silent', action='store_true')
+        p.add_argument('-l', '--ls', action='store_true',
+                help="List dependent nodes.")
+        p.add_argument('-p', '--path', action='store_true',
+                help="List paths to dependent nodes.")
+        p.add_argument('-t', '--tree', action='store_true',
+                help="Dump tree.")
+        p.add_argument('-s', '--start', type=str, nargs='?',
+                help="Start symtype entry/entries.")
+        p.add_argument('symtype', type=str)
+
+    parser_df.add_argument('reference', type=str)
+    parser_df.add_argument('new', type=str)
+    parser_df.add_argument('-s', '--start', type=str, nargs='?',
+                help="Start symtype entry/entries.")
+
+    args = parser.parse_args()
+
+    if args.mode == "index":
+        index(args.symtype, args.output)
+    elif args.mode == "image" or args.mode == "preimage":
+        im(args.symtype, args.ls, args.path, args.tree, args.start, args.mode == "preimage", args.silent)
+    elif args.mode == "diff":
+        diff(args.reference, args.new, args.start)
diff --git a/redhat/kabi/update-kabi b/redhat/kabi/update-kabi
new file mode 100755
index blahblah..blahblah 100755
--- /dev/null
+++ b/redhat/kabi/update-kabi
@@ -0,0 +1,125 @@
+#!/usr/bin/env bash
+
+usage_desc() {
+	sed 's/^\t//' <<EOF
+	Updates the kernel ABI stablelist checksum information and symtypes
+	for architecture ARCH.
+
+	The file updates stablelist definition in \$REDHAT/kabi/*. It does not
+	stage the newly updated stablelist.
+EOF
+}
+
+emit_line_from_Module_symvers() {
+	[ ! -e Module.symvers ] && return 1
+	grep -P "\t$2\t" Module.symvers
+}
+
+emit_line() {
+	local checksum="$1"
+	local symbol="$2"
+	local obj="${3:-unknown}"
+	local macro="${4:-UNKNOWN_MACRO}"
+	local namespace="${5:-}"
+
+	if [ -z "${namespace:-}" ]; then
+		printf "%s\t%s\t%s\t%s\n" $checksum $symbol $obj $macro
+		return
+	fi
+
+	printf "%s\t%s\t%s\t%s\t%s\n" $checksum $symbol $obj $macro $namespace
+}
+
+cb_checksum_new() {
+	local arch="$1"
+	local symbol="$2"
+	local checksum="$3"
+	local file="$4"
+	local symversions="$5"
+	local symtypes="$6"
+	local stable_entry=$REDHAT/kabi/kabi-module/kabi_$arch/$symbol
+	local prev_csum="NULL"
+
+	{
+		printf "#0-\n"
+		printf "#I:0\n"
+		printf "#P:$file\n"
+		if ! emit_line_from_Module_symvers $arch $symbol; then
+			[ ${TEST:-0} -eq 0 ] && return 1
+			emit_line $checksum $symbol null null
+		fi
+	} > $stable_entry
+
+	echo2 "Added symbol \`$sym' for architecture \`$arch' ($checksum)."
+	emit_line_from_Module_symvers $arch $symbol || :
+}
+
+cb_checksum_update() {
+	local arch="$1"
+	local symbol="$2"
+	local checksum="$3"
+	local file="$4"
+	local symversions="$5"
+	local symtypes="$6"
+
+	local prev_index="$7"
+	local prev_csum="$8"
+	#prev_symbol $9 unused
+	local prev_obj="${10}"
+	local prev_macro="${11}"
+	local prev_ns="${12:-}"
+
+	local stable_entry=$REDHAT/kabi/kabi-module/kabi_$arch/$symbol
+	local index=$(grep -v '^#' $stable_entry | wc -l)
+
+	if [ "$prev_csum" = "$checksum" ]; then
+		echo2 "Symbol checksum of \`$sym' for architecture \`$arch' unchanged."
+		return
+	fi
+
+	sed -i -e "s#\#P:.*#\#P:$file#" -e "s/#I:[0-9]\+/#I:$index/" \
+		$stable_entry
+	if ! emit_line_from_Module_symvers $arch $symbol; then
+		emit_line $checksum $symbol $prev_obj $prev_macro $prev_ns
+	fi >> $stable_entry
+
+	echo2 "Updated symbol \`$sym' for architecture \`$arch'" \
+	      "($prev_csum -> $checksum)."
+}
+
+# Called whenever a new symbol checksum is obtained
+# cb_checksum ARCH SYMBOL CHECKSUM FILE SYMVERSIONS_FILE SYMTYPES_FILE
+cb_checksum() {
+	local arch="$1"
+	local symbol="$2"
+	local checksum="$3"
+	local symversions="$5"
+	local symtypes="$6"
+	local stable_entry=$REDHAT/kabi/kabi-module/kabi_$arch/$symbol
+
+	cp $symtypes $REDHAT/kabi/kabi-module/kabi_$arch/.$symbol
+
+	if [ -e $stable_entry ]; then
+		prev_index=$(grep -Po "^#I:\K[0-9]+" $stable_entry)
+		cb_checksum_update "$@" $prev_index $(
+			IFS=' ' grep -v "^#" $stable_entry \
+			| head -n$((${prev_index}+1)) \
+			| tail -n1
+		)
+		return
+	fi
+
+	cb_checksum_new "$@"
+}
+
+# Called whenever the generate command finished successfully
+cb_ready() {
+	# no more symbols, nothing to do
+	:
+}
+
+cd "$(git rev-parse --show-toplevel)"
+REDHAT=${REDHAT:-$(pwd)/redhat/}
+
+# Generate new symbol checksums and symvers files
+. $REDHAT/kabi/symtype-generate

--
https://gitlab.com/cki-project/kernel-ark/-/merge_requests/2021
_______________________________________________
kernel mailing list -- kernel@xxxxxxxxxxxxxxxxxxxxxxx
To unsubscribe send an email to kernel-leave@xxxxxxxxxxxxxxxxxxxxxxx
Fedora Code of Conduct: https://docs.fedoraproject.org/en-US/project/code-of-conduct/
List Guidelines: https://fedoraproject.org/wiki/Mailing_list_guidelines
List Archives: https://lists.fedoraproject.org/archives/list/kernel@xxxxxxxxxxxxxxxxxxxxxxx
Do not reply to spam, report it: https://pagure.io/fedora-infrastructure/new_issue




[Index of Archives]     [Fedora General Discussion]     [Older Fedora Users Archive]     [Fedora Advisory Board]     [Fedora Security]     [Fedora Devel Java]     [Fedora Legacy]     [Fedora Desktop]     [ATA RAID]     [Fedora Marketing]     [Fedora Mentors]     [Fedora Package Announce]     [Fedora Package Review]     [Fedora Music]     [Fedora Packaging]     [Centos]     [Fedora SELinux]     [Coolkey]     [Yum Users]     [Tux]     [Yosemite News]     [KDE Users]     [Fedora Art]     [Fedora Docs]     [USB]     [Asterisk PBX]

  Powered by Linux