From 5f068dd3bc581885d72981395dc489b3e4f1d296 Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Wed, 6 Dec 2023 11:48:06 +0000 Subject: [PATCH] UBUNTU: [Packaging] Remove in-tree abi checks linux-buildinfo packages are now externally compared by swm, with results approving or rejecting updates based on the stable tracker. Those checks also allow hints and overrides to accept intentional changes. Also these are done on the correct pair-wise comparisons, especially when two streams are being cranked. The above eliminates the need to identify previous build abi, download, extract it, vendor it in, and assert it at build time. Signed-off-by: Dimitri John Ledkov --- debian/rules | 8 +- debian/rules.d/0-common-vars.mk | 1 - debian/rules.d/4-checks.mk | 14 +- debian/scripts/checks/abi-check | 139 ------------------ debian/scripts/checks/final-checks | 44 ------ debian/scripts/checks/module-check | 131 ----------------- debian/scripts/misc/getabis | 222 ----------------------------- 7 files changed, 2 insertions(+), 557 deletions(-) delete mode 100755 debian/scripts/checks/abi-check delete mode 100755 debian/scripts/checks/module-check delete mode 100755 debian/scripts/misc/getabis diff --git a/debian/rules b/debian/rules index 1319a5f5d86a..f7d3c10a0bad 100755 --- a/debian/rules +++ b/debian/rules @@ -109,12 +109,6 @@ ifneq ($(do_tools),true) do_tools_host= endif -$(foreach _m,$(all_built-in_dkms_modules), \ - $(if $(filter true,$(do_$(_m))),, \ - $(eval do_$(_m)_disable := $$(shell for m in $$$$(cat $(DROOT)/$(_m)-modules.ignore); do grep -qxF $$$$m $(prev_abidir)/../modules.ignore 2>/dev/null || echo $$$$m >> $(prev_abidir)/../modules.ignore; done)) \ - ) \ -) - # Either tools package needs the common source preparation do_any_tools=$(sort $(filter-out false,$(do_linux_tools) $(do_cloud_tools))) @@ -132,7 +126,7 @@ clean: debian/control debian/canonical-certs.pem debian/canonical-revoked-certs. dh_clean # normal build junk - rm -rf $(DEBIAN)/abi/$(release)-$(revision) + rm -rf $(DEBIAN)/abi rm -rf $(builddir) rm -f $(stampdir)/stamp-* rm -rf debian/linux-*/ diff --git a/debian/rules.d/0-common-vars.mk b/debian/rules.d/0-common-vars.mk index 0f6108d7ca05..d64349e2ab59 100644 --- a/debian/rules.d/0-common-vars.mk +++ b/debian/rules.d/0-common-vars.mk @@ -88,7 +88,6 @@ export gcc?=gcc-13 GCC_BUILD_DEPENDS=\ $(gcc), $(gcc)-aarch64-linux-gnu [arm64] , $(gcc)-arm-linux-gnueabihf [armhf] , $(gcc)-powerpc64le-linux-gnu [ppc64el] , $(gcc)-riscv64-linux-gnu [riscv64] , $(gcc)-s390x-linux-gnu [s390x] , $(gcc)-x86-64-linux-gnu [amd64] , abidir := $(CURDIR)/$(DEBIAN)/__abi.current/$(arch) -prev_abidir := $(CURDIR)/$(DEBIAN)/abi/$(arch) commonconfdir := $(CURDIR)/$(DEBIAN)/config archconfdir := $(CURDIR)/$(DEBIAN)/config/$(arch) sharedconfdir := $(CURDIR)/debian.master/config diff --git a/debian/rules.d/4-checks.mk b/debian/rules.d/4-checks.mk index 6e2d4f8524cf..7cb84e9f71e5 100644 --- a/debian/rules.d/4-checks.mk +++ b/debian/rules.d/4-checks.mk @@ -1,15 +1,3 @@ -# Check ABI for package against last release (if not same abinum) -abi-check-%: $(stampdir)/stamp-install-% - @echo Debug: $@ - $(DROOT)/scripts/checks/abi-check "$*" \ - "$(prev_abidir)" "$(abidir)" $(do_skip_checks) - -# Check the module list against the last release (always) -module-check-%: $(stampdir)/stamp-install-% - @echo Debug: $@ - $(DROOT)/scripts/checks/module-check "$*" \ - "$(prev_abidir)" "$(abidir)" $(do_skip_checks) - # Check the signature of staging modules module-signature-check-%: $(stampdir)/stamp-install-% @echo Debug: $@ @@ -18,7 +6,7 @@ module-signature-check-%: $(stampdir)/stamp-install-% "$(DROOT)/$(mods_extra_pkg_name)-$*" \ $(do_skip_checks) -checks-%: module-check-% module-signature-check-% abi-check-% +checks-%: module-signature-check-% @echo Debug: $@ # Check the config against the known options list. diff --git a/debian/scripts/checks/abi-check b/debian/scripts/checks/abi-check deleted file mode 100755 index f29ce638ff27..000000000000 --- a/debian/scripts/checks/abi-check +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/python3 -# -# Check ABI changes -# -# To skip the ABI checks, add a file -# debian./abi//ignore.abi -# or -# debian./abi//.ignore.abi -# -# To ignore a list of symbols, add the symbols to the file -# debian./abi/abi.ignore -# - -import os -import re -import sys - -def decode_symline(line): - comps = re.sub(r'\s+', ' ', line).split(' ') - if comps[0].startswith('EXPORT_SYMBOL'): - stype, sloc, shash, sname = comps - else: - stype, shash, sname, sloc = comps[1:] - return sname, {'type': stype, 'loc': sloc, 'hash': shash} - -if len(sys.argv) < 4 or len(sys.argv) > 5: - print('Usage: abi-check []') - sys.exit(2) - -flavor, prev_abidir, abidir = sys.argv[1:4] # pylint: disable=W0632 -if len(sys.argv) > 4: - skipabi = sys.argv[4].lower() in ['1', 'true', 'yes'] -else: - skipabi = False - -print('II: Checking ABI for {}...'.format(flavor), end='') - -if ((os.path.exists('{}/ignore.abi'.format(prev_abidir)) or - os.path.exists('{}/{}.ignore.abi'.format(prev_abidir, flavor)))): - print('WW: Explicitly ignoring ABI') - print('II: Done') - sys.exit(0) - -curr_abi = '{}/{}'.format(abidir, flavor) -prev_abi = '{}/{}'.format(prev_abidir, flavor) -if not os.path.exists(curr_abi) or not os.path.exists(prev_abi): - print('II: Previous or current ABI file missing!') - print(' {}'.format(curr_abi)) - print(' {}'.format(prev_abi)) - if skipabi: - print('WW: Explicitly asked to ignore failures') - print('II: Done') - sys.exit(0) - print('EE: Missing ABI file') - sys.exit(1) - -print() - -symbols = {} -symbols_ignore = {} - -# See if we have any ignores -print(' Reading symbols to ignore...', end='') -ignore = 0 -prev_abi_ignore = '{}/../abi.ignore'.format(prev_abidir) -if os.path.exists(prev_abi_ignore): - with open(prev_abi_ignore) as fh: - for sym in fh: - sym = sym.strip() - if sym.startswith('#'): - continue - symbols_ignore[sym] = 1 - ignore += 1 -print('read {} symbols.'.format(ignore)) - -# Read new symbols first -print(' Reading new symbols...', end='') -new_count = 0 -with open('{}/{}'.format(abidir, flavor)) as fh: - for line in fh: - sym, vals = decode_symline(line.strip()) - symbols[sym] = vals - new_count += 1 -print('read {} symbols.'.format(new_count)) - -# Now the old symbols -print(' Reading old symbols...', end='') -old_count = 0 -with open('{}/{}'.format(prev_abidir, flavor)) as fh: - for line in fh: - sym, vals = decode_symline(line.strip()) - if sym not in symbols: - symbols[sym] = {} - symbols[sym]['old'] = vals - old_count += 1 -print('read {} symbols.'.format(old_count)) - -print('II: Checking for ABI changes...') -changed_loc = 0 -changed_type = 0 -error = False -for sym, vals in symbols.items(): - # Ignore new symbols - if 'old' not in vals or 'loc' not in vals: - continue - - # Changes in location don't hurt us, but log it anyway - if vals['loc'] != vals['old']['loc']: - changed_loc += 1 - print(' MOVE : {:40} : {} => {}'.format(sym, vals['old']['loc'], - vals['loc'])) - - # Changes from GPL to non-GPL are bad - if ((vals['old']['type'] == 'EXPORT_SYMBOL_GPL' and - vals['type'] != 'EXPORT_SYMBOL_GPL')): - changed_type += 1 - if sym in symbols_ignore or vals['loc'] in symbols_ignore: - ignored = ' (ignore)' - else: - ignored = '' - error = True - print(' TYPE : {:40} : {} => {}{}'.format(sym, vals['old']['type'], - vals['type'], ignored)) - -if changed_loc > 0: - print('II: {} symbols changed location'.format(changed_loc)) - -if changed_type > 0: - print('II: {} symbols changed export type'.format(changed_type)) - -if error: - if skipabi: - print('WW: Explicitly asked to ignore failures') - else: - print('EE: Symbol types changed') - sys.exit(1) - -print('II: Done') -sys.exit(0) diff --git a/debian/scripts/checks/final-checks b/debian/scripts/checks/final-checks index 4602cb5c34bd..98986b341bda 100755 --- a/debian/scripts/checks/final-checks +++ b/debian/scripts/checks/final-checks @@ -3,12 +3,6 @@ debian="$1" abi="$2" -skipabi=${3:-} -case "${skipabi,,}" in - 1|true|yes) skipabi=true ;; - *) skipabi=false ;; -esac - archs=$(awk '/^Architecture:/ { $1=""; for (i=1; i<=NF; i++) { if ($i != "all") { print $i }}}' debian/control | sort -u) fail=0 @@ -19,41 +13,6 @@ failure() fail=1 } -abi_check() -{ - local abidir="$1" - local arch="$2" - local flavour="$3" - - local abidir="$abidir/$arch" - - if [ ! -f "$abidir/$flavour" ] && \ - [ ! -f "$abidir/$flavour.ignore" ] && \ - [ ! -f "$abidir/ignore" ] - then - failure "$arch/$flavour ABI symbol file missing" - fi - - if [ ! -f "$abidir/$flavour.modules" ] && \ - [ ! -f "$abidir/$flavour.ignore.modules" ] && \ - [ ! -f "$abidir/ignore.modules" ] - then - failure "$arch/$flavour ABI modules file missing" - fi -} - -abi_version="$debian/abi/version" -if ! [ -f "$abi_version" ] -then - failure "$abi_version ABI version file missing" -fi - -version=$(cat "$abi_version") -if [ "$abi" != "$version" ] -then - failure "$abi_version ABI version mismatch ($abi != $version)" -fi - for arch in $archs do if [ ! -f "$debian/rules.d/$arch.mk" ]; then @@ -82,9 +41,6 @@ do failure "'CONFIG_SYSTEM_REVOCATION_KEYS=\"debian/canonical-revoked-certs.pem\"' is required" fi fi - if [ "${skipabi}" = "false" ] ; then - abi_check "$debian/abi" "$arch" "$flavour" - fi done done diff --git a/debian/scripts/checks/module-check b/debian/scripts/checks/module-check deleted file mode 100755 index dcad1b33b45f..000000000000 --- a/debian/scripts/checks/module-check +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/python3 -# -# Check modules changes -# -# To skip the modules check, add a file -# debian./abi//ignore.modules -# or -# debian./abi//.ignore.modules -# -# To ignore a list of modules, add the modules to the file -# debian./abi/modules.ignore -# - -import os -import sys - -if len(sys.argv) < 4 or len(sys.argv) > 5: - print('Usage: module-check []') - sys.exit(2) - -flavor, prev_abidir, abidir = sys.argv[1:4] # pylint: disable=W0632 -if len(sys.argv) > 4: - skipmodule = sys.argv[4].lower() in ['1', 'true', 'yes'] -else: - skipmodule = False - -print('II: Checking modules for {}...'.format(flavor), end='') - -if ((os.path.exists('{}/ignore.modules'.format(prev_abidir)) or - os.path.exists('{}/{}.ignore.modules'.format(prev_abidir, flavor)))): - print('WW: Explicitly ignoring modules') - print('II: Done') - sys.exit(0) - -curr_modules = '{}/{}.modules'.format(abidir, flavor) -prev_modules = '{}/{}.modules'.format(prev_abidir, flavor) -if not os.path.exists(curr_modules) or not os.path.exists(prev_modules): - print('II: Previous or current modules file missing!') - print(' {}'.format(curr_modules)) - print(' {}'.format(prev_modules)) - if skipmodule: - print('WW: Explicitly asked to ignore failures') - print('II: Done') - sys.exit(0) - print('EE: Missing modules file') - sys.exit(1) - -print() - -modules = {} -modules_ignore = {} - -# See if we have any ignores -print(' Reading modules to ignore...', end='') -ignore = 0 -prev_modules_ignore = '{}/../modules.ignore'.format(prev_abidir) -if os.path.exists(prev_modules_ignore): - with open(prev_modules_ignore) as fh: - for mod in fh: - mod = mod.strip() - if mod.startswith('#'): - continue - modules_ignore[mod] = 1 - ignore += 1 -print('read {} modules.'.format(ignore)) - -# Read new modules first -print(' Reading new modules...', end='') -new_count = 0 -for f in (curr_modules, curr_modules + '.builtin'): - if not os.path.exists(f): - continue - with open(f) as fh: - for mod in fh: - mod = mod.strip() - modules[mod] = {'new': 1} - new_count += 1 -print('read {} modules.'.format(new_count)) - -# Now the old modules -print(' Reading old modules...', end='') -old_count = 0 -for f in (prev_modules, prev_modules + '.builtin'): - if not os.path.exists(f): - continue - with open(f) as fh: - for mod in fh: - mod = mod.strip() - if mod not in modules: - modules[mod] = {} - modules[mod]['old'] = 1 - old_count += 1 -print('read {} modules.'.format(old_count)) - -print('II: Checking for modules changes...') -new = 0 -missing = 0 -missing_ignored = 0 -error = False -for mod, vals in modules.items(): - # New modules - if 'old' not in vals: - print(' NEW: {}'.format(mod)) - new += 1 - - # Missing modules - if 'new' not in vals: - missing += 0 - if mod in modules_ignore: - ignored = ' (ignored)' - missing_ignored += 1 - else: - ignored = '' - error = True - print(' MISS : {}{}'.format(mod, ignored)) - -if new > 0: - print('II: {} new modules'.format(new)) - -if missing > 0: - print('II: {} missing modules ({} ignored)'.format(missing, missing_ignored)) - -if error: - if skipmodule: - print('WW: Explicitly asked to ignore failures') - else: - print('EE: Missing modules') - sys.exit(1) - -print('II: Done') -sys.exit(0) diff --git a/debian/scripts/misc/getabis b/debian/scripts/misc/getabis deleted file mode 100755 index 819824f6162c..000000000000 --- a/debian/scripts/misc/getabis +++ /dev/null @@ -1,222 +0,0 @@ -#!/bin/bash - -export LC_ALL=C.UTF-8 - -if [ "$#" = "1" ]; then - set - $(echo "$1" | sed -e 's/-/ /') -fi -if [ "$#" != "2" ]; then - echo "Usage: $0 " 1>&2 - echo "Usage: $0 " 1>&2 - exit 1 -fi - -if [ "$DEBIAN" = "" ]; then - . debian/debian.env -fi - -ver=$1 -revision=$2 -abi=${revision%%.*} - -verabi=$ver-$abi -verfull=$ver-$revision - -WGET="wget --tries=1 --timeout=10 --quiet -c" - -# Check if we use a flat (unversioned) ABI directory -if [ -f "${DEBIAN}/abi/version" ] || \ - grep -qP '^abidir\s+.*/__abi.current/' debian/rules.d/0-common-vars.mk ; then - echo "Using flat ABI directory" - flat_abi=1 - abidir=$(pwd)/${DEBIAN}/abi -else - echo "Using versioned ABI directory" - flat_abi=0 - abidir=$(pwd)/${DEBIAN}/abi/${verfull} -fi - -tmpdir="`pwd`/abi-tmp-$verfull" -origdir="`pwd`" -fwinfo=$abidir/fwinfo - -test -d $tmpdir || mkdir $tmpdir - -package_prefixes() { - : # no longer used ... -} - -getall() { - arch=$1 - shift - - mkdir -p $abidir/$arch - - for sub in $@; do - if [ -f $abidir/$arch/$sub ]; then - echo "Existing $sub($arch)..." - continue - fi - echo "Fetching $sub($arch)..." - getall_set "linux-buildinfo" "$arch" "$sub" || \ - getall_set "linux-image-unsigned linux-modules linux-modules-extra" "$arch" "$sub" || \ - getall_set "linux-image-unsigned linux-modules" "$arch" "$sub" || \ - getall_set "linux-image linux-modules linux-modules-extra" "$arch" "$sub" || \ - getall_set "linux-image linux-modules" "$arch" "$sub" || \ - getall_set "linux-image linux-image-extra" "$arch" "$sub" || \ - getall_set "linux-image" "$arch" "$sub" || \ - { echo "FAILED"; exit 1; } - done -} -getall_set() -{ - prefixes="$1" - arch="$2" - sub="$3" - ( - echo -n " set:" - filenames="" - cd $tmpdir - found=1 - for prefix in $prefixes - do - echo -n " $prefix=" - if [ "$found" = 0 ]; then - echo -n "-" - continue - fi - filename=${prefix}-${verabi}-${sub}_${verfull}_${arch}.deb - for r in "${repo_list[@]}" - do - if ! [ -f $filename ]; then - $WGET $r/$filename - rc="$?" - # If this was not successful or a valid error - # return from the server all bets are off, bail. - [ "$rc" != 0 -a "$rc" != 8 ] && return 2 - fi - if [ -f $filename ]; then - echo -n "y" - filenames="$filenames $filename" - break - fi - done - if [ ! -f "$filename" ]; then - echo -n "n" - found=0 - fi - done - echo "" - if [ "$found" = 0 ]; then - return 1 - fi - echo " extracting..." - for filename in $filenames - do - dpkg-deb --extract $filename tmp - done - # FORM 1: linux-image et al extracted here. - if [ -d tmp/boot ]; then - echo " images..." - find tmp -name "*.ko" | while read f; do - modinfo $f | grep ^firmware >> $fwinfo - done - if [ -f tmp/boot/abi-* ]; then - mv tmp/boot/abi-* $abidir/$arch/$sub - else - echo " NO ABI FILE" - fi - if [ -f tmp/boot/retpoline-* ]; then - mv tmp/boot/retpoline-* $abidir/$arch/$sub.retpoline - else - echo " NO RETPOLINE FILE" - fi - (cd tmp; find lib/modules/$verabi-$sub/kernel -name '*.ko') | \ - sed -e 's/.*\/\([^\/]*\)\.ko/\1/' | sort > \ - $abidir/$arch/$sub.modules - ( - cd tmp; - # Prevent exposing some errors when called by python scripts. SIGPIPE seems to get - # exposed when using the `find ...` form of the command. - ko=$(find lib/modules/$verabi-$sub/kernel \ - -name '*.ko' | head -1) - readelf -p .comment "$ko" | gawk ' - ($1 == "[") { - printf("%s", $3); - for (n=4; n<=NF; n++) { - printf(" %s", $n); - } - print "" - }' | sort -u >$abidir/$arch/$sub.compiler - version=`cat $abidir/$arch/$sub.compiler` - echo " $version" - ) - # FORM 2: moduleinfo packages - # cranky fix -- modinfo supported - else - echo " buildinfo..." - base="tmp/usr/lib/linux/${verabi}-${sub}" - mv "$base/abi" "$abidir/$arch/$sub" - for comp in 'modules' 'retpoline' 'compiler' - do - mv "$base/$comp" "$abidir/$arch/$sub.$comp" - done - if [ -e "${base}"/modules.builtin ] ; then - mv "${base}"/modules.builtin "${abidir}/${arch}/${sub}".modules.builtin - fi - cat "$base/fwinfo" >>"$fwinfo" - if [ -e "${base}"/fwinfo.builtin ] ; then - cat "${base}"/fwinfo.builtin >> "${fwinfo}".builtin - fi - fi - rm -rf tmp $filenames - echo " done" - ) - rc="$?" - if [ "$rc" = 2 ]; then - echo "ERROR: downloads are reporting network failures" 1>&2 - exit 1 - fi - return "$rc" -} - -# MAIN - -# Setup abi directory -rm -rf "${abidir}" -mkdir -p $abidir -echo $abi > $abidir/abiname -if [ ${flat_abi} -eq 1 ] ; then - echo "${verfull}" > "${abidir}"/version -fi - -# NOTE: The flavours are hardcoded, because they may have changed from the -# current build. - -. $DEBIAN/etc/getabis - -# Extract compiler source package version from e.g.: -# GCC: (Ubuntu/Linaro 4.8.2-19ubuntu1) 4.8.2 -compilers=`sed 's/^.*(.* \(.*\)).*$/\1/' $abidir/*/*.compiler | sort -u | wc -l` -if [ "$compilers" != 1 ]; then - echo "WARNING: inconsistent compiler versions detected:" 1>&2 - sort -u $abidir/*/*.compiler | sed 's/^/WARNING: /' 1>&2 -fi - -# Sort fwinfo files -sort < $fwinfo | uniq > fwinfo.tmp -mv fwinfo.tmp $fwinfo -if [ -e "${fwinfo}".builtin ] ; then - sort < "${fwinfo}".builtin | uniq > fwinfo.tmp - mv fwinfo.tmp "${fwinfo}".builtin -fi - -rmdir $tmpdir - -# If this is running in a git repo, add the new ABI directory, remove the old -if [ -d ".git" ]; then - git add "${abidir}" - if [ ${flat_abi} -eq 0 ] ; then - find "$DEBIAN"/abi/* -maxdepth 0 -type d | grep -v "$verfull" | while read f; do git rm -rf "$f"; done - fi -fi