UBUNTU: [Packaging] Remove in-tree abi checks

linux-buildinfo packages are now externally compared by swm, with
results approving or rejecting updates based on the stable
tracker. Those checks also allow hints and overrides to accept
intentional changes.

Also these are done on the correct pair-wise comparisons, especially
when two streams are being cranked.

The above eliminates the need to identify previous build abi,
download, extract it, vendor it in, and assert it at build time.

Signed-off-by: Dimitri John Ledkov <dimitri.ledkov@canonical.com>
This commit is contained in:
Dimitri John Ledkov
2023-12-06 11:48:06 +00:00
committed by Paolo Pisati
parent ac677c9773
commit 5f068dd3bc
7 changed files with 2 additions and 557 deletions
+1 -7
View File
@@ -109,12 +109,6 @@ ifneq ($(do_tools),true)
do_tools_host=
endif
$(foreach _m,$(all_built-in_dkms_modules), \
$(if $(filter true,$(do_$(_m))),, \
$(eval do_$(_m)_disable := $$(shell for m in $$$$(cat $(DROOT)/$(_m)-modules.ignore); do grep -qxF $$$$m $(prev_abidir)/../modules.ignore 2>/dev/null || echo $$$$m >> $(prev_abidir)/../modules.ignore; done)) \
) \
)
# Either tools package needs the common source preparation
do_any_tools=$(sort $(filter-out false,$(do_linux_tools) $(do_cloud_tools)))
@@ -132,7 +126,7 @@ clean: debian/control debian/canonical-certs.pem debian/canonical-revoked-certs.
dh_clean
# normal build junk
rm -rf $(DEBIAN)/abi/$(release)-$(revision)
rm -rf $(DEBIAN)/abi
rm -rf $(builddir)
rm -f $(stampdir)/stamp-*
rm -rf debian/linux-*/
-1
View File
@@ -88,7 +88,6 @@ export gcc?=gcc-13
GCC_BUILD_DEPENDS=\ $(gcc), $(gcc)-aarch64-linux-gnu [arm64] <cross>, $(gcc)-arm-linux-gnueabihf [armhf] <cross>, $(gcc)-powerpc64le-linux-gnu [ppc64el] <cross>, $(gcc)-riscv64-linux-gnu [riscv64] <cross>, $(gcc)-s390x-linux-gnu [s390x] <cross>, $(gcc)-x86-64-linux-gnu [amd64] <cross>,
abidir := $(CURDIR)/$(DEBIAN)/__abi.current/$(arch)
prev_abidir := $(CURDIR)/$(DEBIAN)/abi/$(arch)
commonconfdir := $(CURDIR)/$(DEBIAN)/config
archconfdir := $(CURDIR)/$(DEBIAN)/config/$(arch)
sharedconfdir := $(CURDIR)/debian.master/config
+1 -13
View File
@@ -1,15 +1,3 @@
# Check ABI for package against last release (if not same abinum)
abi-check-%: $(stampdir)/stamp-install-%
@echo Debug: $@
$(DROOT)/scripts/checks/abi-check "$*" \
"$(prev_abidir)" "$(abidir)" $(do_skip_checks)
# Check the module list against the last release (always)
module-check-%: $(stampdir)/stamp-install-%
@echo Debug: $@
$(DROOT)/scripts/checks/module-check "$*" \
"$(prev_abidir)" "$(abidir)" $(do_skip_checks)
# Check the signature of staging modules
module-signature-check-%: $(stampdir)/stamp-install-%
@echo Debug: $@
@@ -18,7 +6,7 @@ module-signature-check-%: $(stampdir)/stamp-install-%
"$(DROOT)/$(mods_extra_pkg_name)-$*" \
$(do_skip_checks)
checks-%: module-check-% module-signature-check-% abi-check-%
checks-%: module-signature-check-%
@echo Debug: $@
# Check the config against the known options list.
-139
View File
@@ -1,139 +0,0 @@
#!/usr/bin/python3
#
# Check ABI changes
#
# To skip the ABI checks, add a file
# debian.<foo>/abi/<arch>/ignore.abi
# or
# debian.<foo>/abi/<arch>/<flavor>.ignore.abi
#
# To ignore a list of symbols, add the symbols to the file
# debian.<foo>/abi/abi.ignore
#
import os
import re
import sys
def decode_symline(line):
comps = re.sub(r'\s+', ' ', line).split(' ')
if comps[0].startswith('EXPORT_SYMBOL'):
stype, sloc, shash, sname = comps
else:
stype, shash, sname, sloc = comps[1:]
return sname, {'type': stype, 'loc': sloc, 'hash': shash}
if len(sys.argv) < 4 or len(sys.argv) > 5:
print('Usage: abi-check <flavor> <prev_abidir> <abidir> [<skipabi>]')
sys.exit(2)
flavor, prev_abidir, abidir = sys.argv[1:4] # pylint: disable=W0632
if len(sys.argv) > 4:
skipabi = sys.argv[4].lower() in ['1', 'true', 'yes']
else:
skipabi = False
print('II: Checking ABI for {}...'.format(flavor), end='')
if ((os.path.exists('{}/ignore.abi'.format(prev_abidir)) or
os.path.exists('{}/{}.ignore.abi'.format(prev_abidir, flavor)))):
print('WW: Explicitly ignoring ABI')
print('II: Done')
sys.exit(0)
curr_abi = '{}/{}'.format(abidir, flavor)
prev_abi = '{}/{}'.format(prev_abidir, flavor)
if not os.path.exists(curr_abi) or not os.path.exists(prev_abi):
print('II: Previous or current ABI file missing!')
print(' {}'.format(curr_abi))
print(' {}'.format(prev_abi))
if skipabi:
print('WW: Explicitly asked to ignore failures')
print('II: Done')
sys.exit(0)
print('EE: Missing ABI file')
sys.exit(1)
print()
symbols = {}
symbols_ignore = {}
# See if we have any ignores
print(' Reading symbols to ignore...', end='')
ignore = 0
prev_abi_ignore = '{}/../abi.ignore'.format(prev_abidir)
if os.path.exists(prev_abi_ignore):
with open(prev_abi_ignore) as fh:
for sym in fh:
sym = sym.strip()
if sym.startswith('#'):
continue
symbols_ignore[sym] = 1
ignore += 1
print('read {} symbols.'.format(ignore))
# Read new symbols first
print(' Reading new symbols...', end='')
new_count = 0
with open('{}/{}'.format(abidir, flavor)) as fh:
for line in fh:
sym, vals = decode_symline(line.strip())
symbols[sym] = vals
new_count += 1
print('read {} symbols.'.format(new_count))
# Now the old symbols
print(' Reading old symbols...', end='')
old_count = 0
with open('{}/{}'.format(prev_abidir, flavor)) as fh:
for line in fh:
sym, vals = decode_symline(line.strip())
if sym not in symbols:
symbols[sym] = {}
symbols[sym]['old'] = vals
old_count += 1
print('read {} symbols.'.format(old_count))
print('II: Checking for ABI changes...')
changed_loc = 0
changed_type = 0
error = False
for sym, vals in symbols.items():
# Ignore new symbols
if 'old' not in vals or 'loc' not in vals:
continue
# Changes in location don't hurt us, but log it anyway
if vals['loc'] != vals['old']['loc']:
changed_loc += 1
print(' MOVE : {:40} : {} => {}'.format(sym, vals['old']['loc'],
vals['loc']))
# Changes from GPL to non-GPL are bad
if ((vals['old']['type'] == 'EXPORT_SYMBOL_GPL' and
vals['type'] != 'EXPORT_SYMBOL_GPL')):
changed_type += 1
if sym in symbols_ignore or vals['loc'] in symbols_ignore:
ignored = ' (ignore)'
else:
ignored = ''
error = True
print(' TYPE : {:40} : {} => {}{}'.format(sym, vals['old']['type'],
vals['type'], ignored))
if changed_loc > 0:
print('II: {} symbols changed location'.format(changed_loc))
if changed_type > 0:
print('II: {} symbols changed export type'.format(changed_type))
if error:
if skipabi:
print('WW: Explicitly asked to ignore failures')
else:
print('EE: Symbol types changed')
sys.exit(1)
print('II: Done')
sys.exit(0)
-44
View File
@@ -3,12 +3,6 @@
debian="$1"
abi="$2"
skipabi=${3:-}
case "${skipabi,,}" in
1|true|yes) skipabi=true ;;
*) skipabi=false ;;
esac
archs=$(awk '/^Architecture:/ { $1=""; for (i=1; i<=NF; i++) { if ($i != "all") { print $i }}}' debian/control | sort -u)
fail=0
@@ -19,41 +13,6 @@ failure()
fail=1
}
abi_check()
{
local abidir="$1"
local arch="$2"
local flavour="$3"
local abidir="$abidir/$arch"
if [ ! -f "$abidir/$flavour" ] && \
[ ! -f "$abidir/$flavour.ignore" ] && \
[ ! -f "$abidir/ignore" ]
then
failure "$arch/$flavour ABI symbol file missing"
fi
if [ ! -f "$abidir/$flavour.modules" ] && \
[ ! -f "$abidir/$flavour.ignore.modules" ] && \
[ ! -f "$abidir/ignore.modules" ]
then
failure "$arch/$flavour ABI modules file missing"
fi
}
abi_version="$debian/abi/version"
if ! [ -f "$abi_version" ]
then
failure "$abi_version ABI version file missing"
fi
version=$(cat "$abi_version")
if [ "$abi" != "$version" ]
then
failure "$abi_version ABI version mismatch ($abi != $version)"
fi
for arch in $archs
do
if [ ! -f "$debian/rules.d/$arch.mk" ]; then
@@ -82,9 +41,6 @@ do
failure "'CONFIG_SYSTEM_REVOCATION_KEYS=\"debian/canonical-revoked-certs.pem\"' is required"
fi
fi
if [ "${skipabi}" = "false" ] ; then
abi_check "$debian/abi" "$arch" "$flavour"
fi
done
done
-131
View File
@@ -1,131 +0,0 @@
#!/usr/bin/python3
#
# Check modules changes
#
# To skip the modules check, add a file
# debian.<foo>/abi/<arch>/ignore.modules
# or
# debian.<foo>/abi/<arch>/<flavor>.ignore.modules
#
# To ignore a list of modules, add the modules to the file
# debian.<foo>/abi/modules.ignore
#
import os
import sys
if len(sys.argv) < 4 or len(sys.argv) > 5:
print('Usage: module-check <flavor> <prev_abidir> <abidir> [<skipmodule>]')
sys.exit(2)
flavor, prev_abidir, abidir = sys.argv[1:4] # pylint: disable=W0632
if len(sys.argv) > 4:
skipmodule = sys.argv[4].lower() in ['1', 'true', 'yes']
else:
skipmodule = False
print('II: Checking modules for {}...'.format(flavor), end='')
if ((os.path.exists('{}/ignore.modules'.format(prev_abidir)) or
os.path.exists('{}/{}.ignore.modules'.format(prev_abidir, flavor)))):
print('WW: Explicitly ignoring modules')
print('II: Done')
sys.exit(0)
curr_modules = '{}/{}.modules'.format(abidir, flavor)
prev_modules = '{}/{}.modules'.format(prev_abidir, flavor)
if not os.path.exists(curr_modules) or not os.path.exists(prev_modules):
print('II: Previous or current modules file missing!')
print(' {}'.format(curr_modules))
print(' {}'.format(prev_modules))
if skipmodule:
print('WW: Explicitly asked to ignore failures')
print('II: Done')
sys.exit(0)
print('EE: Missing modules file')
sys.exit(1)
print()
modules = {}
modules_ignore = {}
# See if we have any ignores
print(' Reading modules to ignore...', end='')
ignore = 0
prev_modules_ignore = '{}/../modules.ignore'.format(prev_abidir)
if os.path.exists(prev_modules_ignore):
with open(prev_modules_ignore) as fh:
for mod in fh:
mod = mod.strip()
if mod.startswith('#'):
continue
modules_ignore[mod] = 1
ignore += 1
print('read {} modules.'.format(ignore))
# Read new modules first
print(' Reading new modules...', end='')
new_count = 0
for f in (curr_modules, curr_modules + '.builtin'):
if not os.path.exists(f):
continue
with open(f) as fh:
for mod in fh:
mod = mod.strip()
modules[mod] = {'new': 1}
new_count += 1
print('read {} modules.'.format(new_count))
# Now the old modules
print(' Reading old modules...', end='')
old_count = 0
for f in (prev_modules, prev_modules + '.builtin'):
if not os.path.exists(f):
continue
with open(f) as fh:
for mod in fh:
mod = mod.strip()
if mod not in modules:
modules[mod] = {}
modules[mod]['old'] = 1
old_count += 1
print('read {} modules.'.format(old_count))
print('II: Checking for modules changes...')
new = 0
missing = 0
missing_ignored = 0
error = False
for mod, vals in modules.items():
# New modules
if 'old' not in vals:
print(' NEW: {}'.format(mod))
new += 1
# Missing modules
if 'new' not in vals:
missing += 0
if mod in modules_ignore:
ignored = ' (ignored)'
missing_ignored += 1
else:
ignored = ''
error = True
print(' MISS : {}{}'.format(mod, ignored))
if new > 0:
print('II: {} new modules'.format(new))
if missing > 0:
print('II: {} missing modules ({} ignored)'.format(missing, missing_ignored))
if error:
if skipmodule:
print('WW: Explicitly asked to ignore failures')
else:
print('EE: Missing modules')
sys.exit(1)
print('II: Done')
sys.exit(0)
-222
View File
@@ -1,222 +0,0 @@
#!/bin/bash
export LC_ALL=C.UTF-8
if [ "$#" = "1" ]; then
set - $(echo "$1" | sed -e 's/-/ /')
fi
if [ "$#" != "2" ]; then
echo "Usage: $0 <version>" 1>&2
echo "Usage: $0 <release> <revision>" 1>&2
exit 1
fi
if [ "$DEBIAN" = "" ]; then
. debian/debian.env
fi
ver=$1
revision=$2
abi=${revision%%.*}
verabi=$ver-$abi
verfull=$ver-$revision
WGET="wget --tries=1 --timeout=10 --quiet -c"
# Check if we use a flat (unversioned) ABI directory
if [ -f "${DEBIAN}/abi/version" ] || \
grep -qP '^abidir\s+.*/__abi.current/' debian/rules.d/0-common-vars.mk ; then
echo "Using flat ABI directory"
flat_abi=1
abidir=$(pwd)/${DEBIAN}/abi
else
echo "Using versioned ABI directory"
flat_abi=0
abidir=$(pwd)/${DEBIAN}/abi/${verfull}
fi
tmpdir="`pwd`/abi-tmp-$verfull"
origdir="`pwd`"
fwinfo=$abidir/fwinfo
test -d $tmpdir || mkdir $tmpdir
package_prefixes() {
: # no longer used ...
}
getall() {
arch=$1
shift
mkdir -p $abidir/$arch
for sub in $@; do
if [ -f $abidir/$arch/$sub ]; then
echo "Existing $sub($arch)..."
continue
fi
echo "Fetching $sub($arch)..."
getall_set "linux-buildinfo" "$arch" "$sub" || \
getall_set "linux-image-unsigned linux-modules linux-modules-extra" "$arch" "$sub" || \
getall_set "linux-image-unsigned linux-modules" "$arch" "$sub" || \
getall_set "linux-image linux-modules linux-modules-extra" "$arch" "$sub" || \
getall_set "linux-image linux-modules" "$arch" "$sub" || \
getall_set "linux-image linux-image-extra" "$arch" "$sub" || \
getall_set "linux-image" "$arch" "$sub" || \
{ echo "FAILED"; exit 1; }
done
}
getall_set()
{
prefixes="$1"
arch="$2"
sub="$3"
(
echo -n " set:"
filenames=""
cd $tmpdir
found=1
for prefix in $prefixes
do
echo -n " $prefix="
if [ "$found" = 0 ]; then
echo -n "-"
continue
fi
filename=${prefix}-${verabi}-${sub}_${verfull}_${arch}.deb
for r in "${repo_list[@]}"
do
if ! [ -f $filename ]; then
$WGET $r/$filename
rc="$?"
# If this was not successful or a valid error
# return from the server all bets are off, bail.
[ "$rc" != 0 -a "$rc" != 8 ] && return 2
fi
if [ -f $filename ]; then
echo -n "y"
filenames="$filenames $filename"
break
fi
done
if [ ! -f "$filename" ]; then
echo -n "n"
found=0
fi
done
echo ""
if [ "$found" = 0 ]; then
return 1
fi
echo " extracting..."
for filename in $filenames
do
dpkg-deb --extract $filename tmp
done
# FORM 1: linux-image et al extracted here.
if [ -d tmp/boot ]; then
echo " images..."
find tmp -name "*.ko" | while read f; do
modinfo $f | grep ^firmware >> $fwinfo
done
if [ -f tmp/boot/abi-* ]; then
mv tmp/boot/abi-* $abidir/$arch/$sub
else
echo " NO ABI FILE"
fi
if [ -f tmp/boot/retpoline-* ]; then
mv tmp/boot/retpoline-* $abidir/$arch/$sub.retpoline
else
echo " NO RETPOLINE FILE"
fi
(cd tmp; find lib/modules/$verabi-$sub/kernel -name '*.ko') | \
sed -e 's/.*\/\([^\/]*\)\.ko/\1/' | sort > \
$abidir/$arch/$sub.modules
(
cd tmp;
# Prevent exposing some errors when called by python scripts. SIGPIPE seems to get
# exposed when using the `find ...` form of the command.
ko=$(find lib/modules/$verabi-$sub/kernel \
-name '*.ko' | head -1)
readelf -p .comment "$ko" | gawk '
($1 == "[") {
printf("%s", $3);
for (n=4; n<=NF; n++) {
printf(" %s", $n);
}
print ""
}' | sort -u >$abidir/$arch/$sub.compiler
version=`cat $abidir/$arch/$sub.compiler`
echo " $version"
)
# FORM 2: moduleinfo packages
# cranky fix -- modinfo supported
else
echo " buildinfo..."
base="tmp/usr/lib/linux/${verabi}-${sub}"
mv "$base/abi" "$abidir/$arch/$sub"
for comp in 'modules' 'retpoline' 'compiler'
do
mv "$base/$comp" "$abidir/$arch/$sub.$comp"
done
if [ -e "${base}"/modules.builtin ] ; then
mv "${base}"/modules.builtin "${abidir}/${arch}/${sub}".modules.builtin
fi
cat "$base/fwinfo" >>"$fwinfo"
if [ -e "${base}"/fwinfo.builtin ] ; then
cat "${base}"/fwinfo.builtin >> "${fwinfo}".builtin
fi
fi
rm -rf tmp $filenames
echo " done"
)
rc="$?"
if [ "$rc" = 2 ]; then
echo "ERROR: downloads are reporting network failures" 1>&2
exit 1
fi
return "$rc"
}
# MAIN
# Setup abi directory
rm -rf "${abidir}"
mkdir -p $abidir
echo $abi > $abidir/abiname
if [ ${flat_abi} -eq 1 ] ; then
echo "${verfull}" > "${abidir}"/version
fi
# NOTE: The flavours are hardcoded, because they may have changed from the
# current build.
. $DEBIAN/etc/getabis
# Extract compiler source package version from e.g.:
# GCC: (Ubuntu/Linaro 4.8.2-19ubuntu1) 4.8.2
compilers=`sed 's/^.*(.* \(.*\)).*$/\1/' $abidir/*/*.compiler | sort -u | wc -l`
if [ "$compilers" != 1 ]; then
echo "WARNING: inconsistent compiler versions detected:" 1>&2
sort -u $abidir/*/*.compiler | sed 's/^/WARNING: /' 1>&2
fi
# Sort fwinfo files
sort < $fwinfo | uniq > fwinfo.tmp
mv fwinfo.tmp $fwinfo
if [ -e "${fwinfo}".builtin ] ; then
sort < "${fwinfo}".builtin | uniq > fwinfo.tmp
mv fwinfo.tmp "${fwinfo}".builtin
fi
rmdir $tmpdir
# If this is running in a git repo, add the new ABI directory, remove the old
if [ -d ".git" ]; then
git add "${abidir}"
if [ ${flat_abi} -eq 0 ] ; then
find "$DEBIAN"/abi/* -maxdepth 0 -type d | grep -v "$verfull" | while read f; do git rm -rf "$f"; done
fi
fi