From 63a035ca10c1e30fdcc1f10b64ddfee8b05fec90 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 2 Jan 2024 10:56:35 +0100 Subject: [PATCH] UBUNTU: [Packaging] update annotations scripts BugLink: https://bugs.launchpad.net/bugs/1786013 Signed-off-by: Andrea Righi --- debian/scripts/misc/annotations | 288 ++-------------- debian/scripts/misc/kconfig/annotations.py | 332 ++++++++++++------- debian/scripts/misc/kconfig/run.py | 365 +++++++++++++++++++++ debian/scripts/misc/kconfig/utils.py | 20 ++ debian/scripts/misc/kconfig/version.py | 10 + 5 files changed, 625 insertions(+), 390 deletions(-) create mode 100644 debian/scripts/misc/kconfig/run.py create mode 100644 debian/scripts/misc/kconfig/utils.py create mode 100644 debian/scripts/misc/kconfig/version.py diff --git a/debian/scripts/misc/annotations b/debian/scripts/misc/annotations index 86d858611780..3a05fe96343e 100755 --- a/debian/scripts/misc/annotations +++ b/debian/scripts/misc/annotations @@ -1,274 +1,34 @@ #!/usr/bin/env python3 # -*- mode: python -*- -# Manage Ubuntu kernel .config and annotations -# Copyright © 2022 Canonical Ltd. + +# This file is not installed; it's just to run annotations from inside a source +# distribution without installing it in the system. import sys + +# Prevent generating .pyc files on import +# +# We may end up adding these files to our git repos by mistake, so simply +# prevent generating them in advance. +# +# There's a tiny performance penalty with this, because python needs to +# re-generate the bytecode on-the-fly every time the script is executed, but +# this overhead is absolutely negligible compared the rest of the kernel build +# time. sys.dont_write_bytecode = True -import os -import argparse -import json -from signal import signal, SIGPIPE, SIG_DFL -from kconfig.annotations import Annotation, KConfig - -VERSION = '0.1' - -SKIP_CONFIGS = ( - # CONFIG_VERSION_SIGNATURE is dynamically set during the build - 'CONFIG_VERSION_SIGNATURE', - # Allow to use a different versions of toolchain tools - 'CONFIG_GCC_VERSION', - 'CONFIG_CC_VERSION_TEXT', - 'CONFIG_AS_VERSION', - 'CONFIG_LD_VERSION', - 'CONFIG_LLD_VERSION', - 'CONFIG_CLANG_VERSION', - 'CONFIG_PAHOLE_VERSION', - 'CONFIG_RUSTC_VERSION_TEXT', - 'CONFIG_BINDGEN_VERSION_TEXT', -) +import os # noqa: E402 Import not at top of file +from kconfig import run # noqa: E402 Import not at top of file -def make_parser(): - parser = argparse.ArgumentParser( - description='Manage Ubuntu kernel .config and annotations', - ) - parser.add_argument('--version', '-v', action='version', version=f'%(prog)s {VERSION}') - - parser.add_argument('--file', '-f', action='store', - help='Pass annotations or .config file to be parsed') - parser.add_argument('--arch', '-a', action='store', - help='Select architecture') - parser.add_argument('--flavour', '-l', action='store', - help='Select flavour (default is "generic")') - parser.add_argument('--config', '-c', action='store', - help='Select a specific config option') - parser.add_argument('--query', '-q', action='store_true', - help='Query annotations') - parser.add_argument('--note', '-n', action='store', - help='Write a specific note to a config option in annotations') - parser.add_argument('--autocomplete', action='store_true', - help='Enable config bash autocomplete: `source <(annotations --autocomplete)`') - parser.add_argument('--source', '-t', action='store_true', - help='Jump to a config definition in the kernel source code') - - ga = parser.add_argument_group(title='Action').add_mutually_exclusive_group(required=False) - ga.add_argument('--write', '-w', action='store', - metavar='VALUE', dest='value', - help='Set a specific config value in annotations (use \'null\' to remove)') - ga.add_argument('--export', '-e', action='store_true', - help='Convert annotations to .config format') - ga.add_argument('--import', '-i', action='store', - metavar="FILE", dest='import_file', - help='Import a full .config for a specific arch and flavour into annotations') - ga.add_argument('--update', '-u', action='store', - metavar="FILE", dest='update_file', - help='Import a partial .config into annotations (only resync configs specified in FILE)') - ga.add_argument('--check', '-k', action='store', - metavar="FILE", dest='check_file', - help='Validate kernel .config with annotations') - return parser +# Update PATH to make sure that annotations can be executed directly from the +# source directory. +def update_path(): + script_dir = os.path.dirname(os.path.abspath(__file__)) + current_path = os.environ.get("PATH", "") + new_path = f"{script_dir}:{current_path}" + os.environ["PATH"] = new_path -_ARGPARSER = make_parser() - - -def arg_fail(message): - print(message) - _ARGPARSER.print_usage() - sys.exit(1) - - -def print_result(config, res): - if res is not None and config not in res: - res = {config or '*': res} - print(json.dumps(res, indent=4)) - - -def do_query(args): - if args.arch is None and args.flavour is not None: - arg_fail('error: --flavour requires --arch') - a = Annotation(args.file) - res = a.search_config(config=args.config, arch=args.arch, flavour=args.flavour) - print_result(args.config, res) - - -def do_autocomplete(args): - a = Annotation(args.file) - res = (c.removeprefix('CONFIG_') for c in a.search_config()) - res_str = ' '.join(res) - print(f'complete -W "{res_str}" annotations') - - -def do_source(args): - if args.config is None: - arg_fail('error: --source requires --config') - if not os.path.exists('tags'): - print('tags not found in the current directory, try: `make tags`') - sys.exit(1) - os.system(f'vim -t {args.config}') - - -def do_note(args): - if args.config is None: - arg_fail('error: --note requires --config') - - # Set the note in annotations - a = Annotation(args.file) - a.set(args.config, note=args.note) - - # Save back to annotations - a.save(args.file) - - # Query and print back the value - a = Annotation(args.file) - res = a.search_config(config=args.config) - print_result(args.config, res) - - -def do_write(args): - if args.config is None: - arg_fail('error: --write requires --config') - - # Set the value in annotations ('null' means remove) - a = Annotation(args.file) - if args.value == 'null': - a.remove(args.config, arch=args.arch, flavour=args.flavour) - else: - a.set(args.config, arch=args.arch, flavour=args.flavour, value=args.value, note=args.note) - - # Save back to annotations - a.save(args.file) - - # Query and print back the value - a = Annotation(args.file) - res = a.search_config(config=args.config) - print_result(args.config, res) - - -def do_export(args): - if args.arch is None: - arg_fail('error: --export requires --arch') - a = Annotation(args.file) - conf = a.search_config(config=args.config, arch=args.arch, flavour=args.flavour) - if conf: - print(a.to_config(conf)) - - -def do_import(args): - if args.arch is None: - arg_fail('error: --arch is required with --import') - if args.flavour is None: - arg_fail('error: --flavour is required with --import') - if args.config is not None: - arg_fail('error: --config cannot be used with --import (try --update)') - - # Merge with the current annotations - a = Annotation(args.file) - c = KConfig(args.import_file) - a.update(c, arch=args.arch, flavour=args.flavour) - - # Save back to annotations - a.save(args.file) - - -def do_update(args): - if args.arch is None: - arg_fail('error: --arch is required with --update') - - # Merge with the current annotations - a = Annotation(args.file) - c = KConfig(args.update_file) - if args.config is None: - configs = list(set(c.config.keys()) - set(SKIP_CONFIGS)) - if configs: - a.update(c, arch=args.arch, flavour=args.flavour, configs=configs) - - # Save back to annotations - a.save(args.file) - - -def do_check(args): - # Determine arch and flavour - if args.arch is None: - arg_fail('error: --arch is required with --check') - - print(f"check-config: loading annotations from {args.file}") - total = good = ret = 0 - - # Load annotations settings - a = Annotation(args.file) - a_configs = a.search_config(arch=args.arch, flavour=args.flavour).keys() - - # Parse target .config - c = KConfig(args.check_file) - c_configs = c.config.keys() - - # Validate .config against annotations - for conf in sorted(a_configs | c_configs): - if conf in SKIP_CONFIGS: - continue - entry = a.search_config(config=conf, arch=args.arch, flavour=args.flavour) - expected = entry[conf] if entry else '-' - value = c.config[conf] if conf in c.config else '-' - if value != expected: - policy = a.config[conf] if conf in a.config else 'undefined' - if 'policy' in policy: - policy = f"policy<{policy['policy']}>" - print(f"check-config: FAIL: ({value} != {expected}): {conf} {policy})") - ret = 1 - else: - good += 1 - total += 1 - - print(f"check-config: {good}/{total} checks passed -- exit {ret}") - sys.exit(ret) - - -def autodetect_annotations(args): - if args.file: - return - # If --file/-f isn't specified try to automatically determine the right - # location of the annotations file looking at debian/debian.env. - try: - with open('debian/debian.env', 'rt', encoding='utf-8') as fd: - args.file = fd.read().rstrip().split('=')[1] + '/config/annotations' - except (FileNotFoundError, IndexError): - arg_fail('error: could not determine DEBDIR, try using: --file/-f') - - -def main(): - # Prevent broken pipe errors when showing output in pipe to other tools - # (less for example) - signal(SIGPIPE, SIG_DFL) - - # Main annotations program - args = _ARGPARSER.parse_args() - autodetect_annotations(args) - - if args.config and not args.config.startswith('CONFIG_'): - args.config = 'CONFIG_' + args.config - - if args.value: - do_write(args) - elif args.note: - do_note(args) - elif args.export: - do_export(args) - elif args.import_file: - do_import(args) - elif args.update_file: - do_update(args) - elif args.check_file: - do_check(args) - elif args.autocomplete: - do_autocomplete(args) - elif args.source: - do_source(args) - else: - do_query(args) - - -if __name__ == '__main__': - main() +update_path() +exit(run.main()) diff --git a/debian/scripts/misc/kconfig/annotations.py b/debian/scripts/misc/kconfig/annotations.py index e9bd5e03ead5..6e903f22faa6 100644 --- a/debian/scripts/misc/kconfig/annotations.py +++ b/debian/scripts/misc/kconfig/annotations.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- mode: python -*- # python module to manage Ubuntu kernel .config and annotations # Copyright © 2022 Canonical Ltd. @@ -12,26 +11,29 @@ from abc import abstractmethod from ast import literal_eval from os.path import dirname, abspath +from kconfig.version import ANNOTATIONS_FORMAT_VERSION -class Config(): - def __init__(self, fname): + +class Config: + def __init__(self, fname, do_include=True): """ Basic configuration file object """ self.fname = fname self.config = {} + self.do_include = do_include raw_data = self._load(fname) self._parse(raw_data) @staticmethod def _load(fname: str) -> str: - with open(fname, 'rt', encoding='utf-8') as fd: + with open(fname, "rt", encoding="utf-8") as fd: data = fd.read() return data.rstrip() def __str__(self): - """ Return a JSON representation of the config """ + """Return a JSON representation of the config""" return json.dumps(self.config, indent=4) @abstractmethod @@ -44,14 +46,15 @@ class KConfig(Config): Parse a .config file, individual config options can be accessed via .config[] """ + def _parse(self, data: str): self.config = {} for line in data.splitlines(): - m = re.match(r'^# (CONFIG_.*) is not set$', line) + m = re.match(r"^# (CONFIG_.*) is not set$", line) if m: self.config[m.group(1)] = literal_eval("'n'") continue - m = re.match(r'^(CONFIG_[A-Za-z0-9_]+)=(.*)$', line) + m = re.match(r"^(CONFIG_[A-Za-z0-9_]+)=(.*)$", line) if m: self.config[m.group(1)] = literal_eval("'" + m.group(2) + "'") continue @@ -61,12 +64,13 @@ class Annotation(Config): """ Parse body of annotations file """ + def _parse_body(self, data: str, parent=True): for line in data.splitlines(): # Replace tabs with spaces, squeeze multiple into singles and # remove leading and trailing spaces - line = line.replace('\t', ' ') - line = re.sub(r' +', ' ', line) + line = line.replace("\t", " ") + line = re.sub(r" +", " ", line) line = line.strip() # Ignore empty lines @@ -74,12 +78,12 @@ class Annotation(Config): continue # Catpure flavors of included files - if line.startswith('# FLAVOUR: '): - self.include_flavour += line.split(' ')[2:] + if line.startswith("# FLAVOUR: "): + self.include_flavour += line.split(" ")[2:] continue # Ignore comments - if line.startswith('#'): + if line.startswith("#"): continue # Handle includes (recursively) @@ -87,46 +91,59 @@ class Annotation(Config): if m: if parent: self.include.append(m.group(1)) - include_fname = dirname(abspath(self.fname)) + '/' + m.group(1) - include_data = self._load(include_fname) - self._parse_body(include_data, parent=False) + if self.do_include: + include_fname = dirname(abspath(self.fname)) + "/" + m.group(1) + include_data = self._load(include_fname) + self._parse_body(include_data, parent=False) continue # Handle policy and note lines - if re.match(r'.* (policy|note)<', line): + if re.match(r".* (policy|note)<", line): try: - conf = line.split(' ')[0] + conf = line.split(" ")[0] if conf in self.config: entry = self.config[conf] else: - entry = {'policy': {}} + entry = {"policy": {}} match = False - m = re.match(r'.* policy<(.*?)>', line) + m = re.match(r".* policy<(.*?)>", line) if m: match = True - try: - entry['policy'] |= literal_eval(m.group(1)) - except TypeError: - entry['policy'] = {**entry['policy'], **literal_eval(m.group(1))} + # Update the previous entry considering potential overrides: + # - if the new entry is adding a rule for a new + # arch/flavour, simply add that + # - if the new entry is overriding a previous + # arch-flavour item, then overwrite that item + # - if the new entry is overriding a whole arch, then + # remove all the previous flavour rules of that arch + new_entry = literal_eval(m.group(1)) + for key in new_entry: + if key in self.arch: + for flavour_key in list(entry["policy"].keys()): + if flavour_key.startswith(key): + del entry["policy"][flavour_key] + entry["policy"][key] = new_entry[key] + else: + entry["policy"][key] = new_entry[key] - m = re.match(r'.* note<(.*?)>', line) + m = re.match(r".* note<(.*?)>", line) if m: - entry['oneline'] = match + entry["oneline"] = match match = True - entry['note'] = "'" + m.group(1).replace("'", '') + "'" + entry["note"] = "'" + m.group(1).replace("'", "") + "'" if not match: - raise SyntaxError('syntax error') + raise SyntaxError("syntax error") self.config[conf] = entry except Exception as e: - raise SyntaxError(str(e) + f', line = {line}') from e + raise SyntaxError(str(e) + f", line = {line}") from e continue # Invalid line - raise SyntaxError(f'invalid line: {line}') + raise SyntaxError(f"invalid line: {line}") - def _parse(self, data: str): + def _legacy_parse(self, data: str): """ Parse main annotations file, individual config options can be accessed via self.config[] @@ -136,35 +153,86 @@ class Annotation(Config): self.flavour = [] self.flavour_dep = {} self.include = [] - self.header = '' + self.header = "" self.include_flavour = [] # Parse header (only main header will considered, headers in includes # will be treated as comments) for line in data.splitlines(): - if re.match(r'^#.*', line): - m = re.match(r'^# ARCH: (.*)', line) + if re.match(r"^#.*", line): + m = re.match(r"^# ARCH: (.*)", line) if m: - self.arch = list(m.group(1).split(' ')) - m = re.match(r'^# FLAVOUR: (.*)', line) + self.arch = list(m.group(1).split(" ")) + m = re.match(r"^# FLAVOUR: (.*)", line) if m: - self.flavour = list(m.group(1).split(' ')) - m = re.match(r'^# FLAVOUR_DEP: (.*)', line) + self.flavour = list(m.group(1).split(" ")) + m = re.match(r"^# FLAVOUR_DEP: (.*)", line) if m: self.flavour_dep = literal_eval(m.group(1)) self.header += line + "\n" else: break - # Parse body (handle includes recursively) + # Return an error if architectures are not defined + if not self.arch: + raise SyntaxError("ARCH not defined in annotations") + # Return an error if flavours are not defined + if not self.flavour: + raise SyntaxError("FLAVOUR not defined in annotations") + + # Parse body self._parse_body(data) # Sanity check: Verify that all FLAVOUR_DEP flavors are valid - for src, tgt in self.flavour_dep.items(): - if src not in self.flavour: - raise SyntaxError(f'Invalid source flavour in FLAVOUR_DEP: {src}') - if tgt not in self.include_flavour: - raise SyntaxError(f'Invalid target flavour in FLAVOUR_DEP: {tgt}') + if self.do_include: + for src, tgt in self.flavour_dep.items(): + if src not in self.flavour: + raise SyntaxError(f"Invalid source flavour in FLAVOUR_DEP: {src}") + if tgt not in self.include_flavour: + raise SyntaxError(f"Invalid target flavour in FLAVOUR_DEP: {tgt}") + + def _json_parse(self, data, is_included=False): + data = json.loads(data) + + # Check if version is supported + version = data["attributes"]["_version"] + if version > ANNOTATIONS_FORMAT_VERSION: + raise SyntaxError(f"annotations format version {version} not supported") + + # Check for top-level annotations vs imported annotations + if not is_included: + self.config = data["config"] + self.arch = data["attributes"]["arch"] + self.flavour = data["attributes"]["flavour"] + self.flavour_dep = data["attributes"]["flavour_dep"] + self.include = data["attributes"]["include"] + self.include_flavour = [] + else: + # We are procesing an imported annotations, so merge all the + # configs and attributes. + try: + self.config = data["config"] | self.config + except TypeError: + self.config = {**self.config, **data["config"]} + self.arch = list(set(self.arch) | set(data["attributes"]["arch"])) + self.flavour = list(set(self.flavour) | set(data["attributes"]["flavour"])) + self.include_flavour = list(set(self.include_flavour) | set(data["attributes"]["flavour"])) + self.flavour_dep = self.flavour_dep | data["attributes"]["flavour_dep"] + + # Handle recursive inclusions + if self.do_include: + for f in data["attributes"]["include"]: + include_fname = dirname(abspath(self.fname)) + "/" + f + data = self._load(include_fname) + self._json_parse(data, is_included=True) + + def _parse(self, data: str): + # Try to parse the legacy format first, otherwise use the new JSON + # format. + try: + self._legacy_parse(data) + except SyntaxError: + self._json_parse(data, is_included=False) def _remove_entry(self, config: str): if self.config[config]: @@ -175,34 +243,40 @@ class Annotation(Config): return if arch is not None: if flavour is not None: - flavour = f'{arch}-{flavour}' + flavour = f"{arch}-{flavour}" else: flavour = arch - del self.config[config]['policy'][flavour] - if not self.config[config]['policy']: + del self.config[config]["policy"][flavour] + if not self.config[config]["policy"]: self._remove_entry(config) else: self._remove_entry(config) - def set(self, config: str, arch: str = None, flavour: str = None, - value: str = None, note: str = None): + def set( + self, + config: str, + arch: str = None, + flavour: str = None, + value: str = None, + note: str = None, + ): if value is not None: if config not in self.config: - self.config[config] = {'policy': {}} + self.config[config] = {"policy": {}} if arch is not None: if flavour is not None: - flavour = f'{arch}-{flavour}' + flavour = f"{arch}-{flavour}" else: flavour = arch - self.config[config]['policy'][flavour] = value + self.config[config]["policy"][flavour] = value else: for a in self.arch: - self.config[config]['policy'][a] = value + self.config[config]["policy"][a] = value if note is not None: - self.config[config]['note'] = "'" + note.replace("'", '') + "'" + self.config[config]["note"] = "'" + note.replace("'", "") + "'" def update(self, c: KConfig, arch: str, flavour: str = None, configs: list = None): - """ Merge configs from a Kconfig object into Annotation object """ + """Merge configs from a Kconfig object into Annotation object""" # Determine if we need to import all configs or a single config if not configs: @@ -210,72 +284,75 @@ class Annotation(Config): try: configs |= self.search_config(arch=arch, flavour=flavour).keys() except TypeError: - configs = {**configs, **self.search_config(arch=arch, flavour=flavour).keys()} + configs = { + **configs, + **self.search_config(arch=arch, flavour=flavour).keys(), + } # Import configs from the Kconfig object into Annotations flavour_arg = flavour if flavour is not None: - flavour = arch + f'-{flavour}' + flavour = arch + f"-{flavour}" else: flavour = arch for conf in configs: if conf in c.config: val = c.config[conf] else: - val = '-' + val = "-" if conf in self.config: - if 'policy' in self.config[conf]: + if "policy" in self.config[conf]: # Add a TODO if a config with a note is changing and print # a warning old_val = self.search_config(config=conf, arch=arch, flavour=flavour_arg) if old_val: old_val = old_val[conf] if val != old_val and "note" in self.config[conf]: - self.config[conf]['note'] = "TODO: update note" + self.config[conf]["note"] = "TODO: update note" print(f"WARNING: {conf} changed from {old_val} to {val}, updating note") - self.config[conf]['policy'][flavour] = val + self.config[conf]["policy"][flavour] = val else: - self.config[conf]['policy'] = {flavour: val} + self.config[conf]["policy"] = {flavour: val} else: - self.config[conf] = {'policy': {flavour: val}} + self.config[conf] = {"policy": {flavour: val}} def _compact(self): # Try to remove redundant settings: if the config value of a flavour is # the same as the one of the main arch simply drop it. for conf in self.config.copy(): - if 'policy' not in self.config[conf]: + if "policy" not in self.config[conf]: continue for flavour in self.flavour: - if flavour not in self.config[conf]['policy']: + if flavour not in self.config[conf]["policy"]: continue - m = re.match(r'^(.*?)-(.*)$', flavour) + m = re.match(r"^(.*?)-(.*)$", flavour) if not m: continue arch = m.group(1) - if arch in self.config[conf]['policy']: - if self.config[conf]['policy'][flavour] == self.config[conf]['policy'][arch]: - del self.config[conf]['policy'][flavour] + if arch in self.config[conf]["policy"]: + if self.config[conf]["policy"][flavour] == self.config[conf]["policy"][arch]: + del self.config[conf]["policy"][flavour] continue if flavour not in self.flavour_dep: continue generic = self.flavour_dep[flavour] - if generic in self.config[conf]['policy']: - if self.config[conf]['policy'][flavour] == self.config[conf]['policy'][generic]: - del self.config[conf]['policy'][flavour] + if generic in self.config[conf]["policy"]: + if self.config[conf]["policy"][flavour] == self.config[conf]["policy"][generic]: + del self.config[conf]["policy"][flavour] continue # Remove rules for flavours / arches that are not supported (not # listed in the annotations header). - for flavour in self.config[conf]['policy'].copy(): + for flavour in self.config[conf]["policy"].copy(): if flavour not in list(set(self.arch + self.flavour)): - del self.config[conf]['policy'][flavour] + del self.config[conf]["policy"][flavour] # Remove configs that are all undefined across all arches/flavours # (unless we have includes) if not self.include: - if 'policy' in self.config[conf]: - if list(set(self.config[conf]['policy'].values())) == ['-']: - self.config[conf]['policy'] = {} + if "policy" in self.config[conf]: + if list(set(self.config[conf]["policy"].values())) == ["-"]: + self.config[conf]["policy"] = {} # Drop empty rules - if not self.config[conf]['policy']: + if not self.config[conf]["policy"]: del self.config[conf] else: # Compact same value across all flavour within the same arch @@ -283,16 +360,16 @@ class Annotation(Config): arch_flavours = [i for i in self.flavour if i.startswith(arch)] value = None for flavour in arch_flavours: - if flavour not in self.config[conf]['policy']: + if flavour not in self.config[conf]["policy"]: break if value is None: - value = self.config[conf]['policy'][flavour] - elif value != self.config[conf]['policy'][flavour]: + value = self.config[conf]["policy"][flavour] + elif value != self.config[conf]["policy"][flavour]: break else: for flavour in arch_flavours: - del self.config[conf]['policy'][flavour] - self.config[conf]['policy'][arch] = value + del self.config[conf]["policy"][flavour] + self.config[conf]["policy"][arch] = value # After the first round of compaction we may end up having configs that # are undefined across all arches, so do another round of compaction to # drop these settings that are not needed anymore @@ -300,34 +377,34 @@ class Annotation(Config): if not self.include: for conf in self.config.copy(): # Remove configs that are all undefined across all arches/flavours - if 'policy' in self.config[conf]: - if list(set(self.config[conf]['policy'].values())) == ['-']: - self.config[conf]['policy'] = {} + if "policy" in self.config[conf]: + if list(set(self.config[conf]["policy"].values())) == ["-"]: + self.config[conf]["policy"] = {} # Drop empty rules - if not self.config[conf]['policy']: + if not self.config[conf]["policy"]: del self.config[conf] @staticmethod def _sorted(config): - """ Sort configs alphabetically but return configs with a note first """ + """Sort configs alphabetically but return configs with a note first""" w_note = [] wo_note = [] for c in sorted(config): - if 'note' in config[c]: + if "note" in config[c]: w_note.append(c) else: wo_note.append(c) return w_note + wo_note def save(self, fname: str): - """ Save annotations data to the annotation file """ + """Save annotations data to the annotation file""" # Compact annotations structure self._compact() # Save annotations to disk - with tempfile.NamedTemporaryFile(mode='w+t', delete=False) as tmp: + with tempfile.NamedTemporaryFile(mode="w+t", delete=False) as tmp: # Write header - tmp.write(self.header + '\n') + tmp.write(self.header + "\n") # Write includes for i in self.include: @@ -344,40 +421,43 @@ class Annotation(Config): marker = False for conf in self._sorted(self.config): new_val = self.config[conf] - if 'policy' not in new_val: + if "policy" not in new_val: continue # If new_val is a subset of old_val, skip it unless there are # new notes that are different than the old ones. old_val = tmp_a.config.get(conf) - if old_val and 'policy' in old_val: + if old_val and "policy" in old_val: try: - can_skip = old_val['policy'] == old_val['policy'] | new_val['policy'] + can_skip = old_val["policy"] == old_val["policy"] | new_val["policy"] except TypeError: - can_skip = old_val['policy'] == {**old_val['policy'], **new_val['policy']} + can_skip = old_val["policy"] == { + **old_val["policy"], + **new_val["policy"], + } if can_skip: - if 'note' not in new_val: + if "note" not in new_val: continue - if 'note' in old_val and 'note' in new_val: - if old_val['note'] == new_val['note']: + if "note" in old_val and "note" in new_val: + if old_val["note"] == new_val["note"]: continue # Write out the policy (and note) line(s) - val = dict(sorted(new_val['policy'].items())) + val = dict(sorted(new_val["policy"].items())) line = f"{conf : <47} policy<{val}>" - if 'note' in new_val: - val = new_val['note'] - if new_val.get('oneline', False): + if "note" in new_val: + val = new_val["note"] + if new_val.get("oneline", False): # Single line - line += f' note<{val}>' + line += f" note<{val}>" else: # Separate policy and note lines, # followed by an empty line - line += f'\n{conf : <47} note<{val}>\n' + line += f"\n{conf : <47} note<{val}>\n" elif not marker: # Write out a marker indicating the start of annotations # without notes - tmp.write('\n# ---- Annotations without notes ----\n\n') + tmp.write("\n# ---- Annotations without notes ----\n\n") marker = True tmp.write(line + "\n") @@ -386,10 +466,10 @@ class Annotation(Config): shutil.move(tmp.name, fname) def search_config(self, config: str = None, arch: str = None, flavour: str = None) -> dict: - """ Return config value of a specific config option or architecture """ + """Return config value of a specific config option or architecture""" if flavour is None: - flavour = 'generic' - flavour = f'{arch}-{flavour}' + flavour = "generic" + flavour = f"{arch}-{flavour}" if flavour in self.flavour_dep: generic = self.flavour_dep[flavour] else: @@ -401,14 +481,14 @@ class Annotation(Config): # Get config options of a specific architecture ret = {} for c, val in self.config.items(): - if 'policy' not in val: + if "policy" not in val: continue - if flavour in val['policy']: - ret[c] = val['policy'][flavour] - elif generic != flavour and generic in val['policy']: - ret[c] = val['policy'][generic] - elif arch in val['policy']: - ret[c] = val['policy'][arch] + if flavour in val["policy"]: + ret[c] = val["policy"][flavour] + elif generic != flavour and generic in val["policy"]: + ret[c] = val["policy"][generic] + elif arch in val["policy"]: + ret[c] = val["policy"][arch] return ret if config is not None and arch is None: # Get a specific config option for all architectures @@ -416,24 +496,24 @@ class Annotation(Config): if config is not None and arch is not None: # Get a specific config option for a specific architecture if config in self.config: - if 'policy' in self.config[config]: - if flavour in self.config[config]['policy']: - return {config: self.config[config]['policy'][flavour]} - if generic != flavour and generic in self.config[config]['policy']: - return {config: self.config[config]['policy'][generic]} - if arch in self.config[config]['policy']: - return {config: self.config[config]['policy'][arch]} + if "policy" in self.config[config]: + if flavour in self.config[config]["policy"]: + return {config: self.config[config]["policy"][flavour]} + if generic != flavour and generic in self.config[config]["policy"]: + return {config: self.config[config]["policy"][generic]} + if arch in self.config[config]["policy"]: + return {config: self.config[config]["policy"][arch]} return None @staticmethod def to_config(data: dict) -> str: - """ Convert annotations data to .config format """ - s = '' + """Convert annotations data to .config format""" + s = "" for c in data: v = data[c] - if v == 'n': + if v == "n": s += f"# {c} is not set\n" - elif v == '-': + elif v == "-": pass else: s += f"{c}={v}\n" diff --git a/debian/scripts/misc/kconfig/run.py b/debian/scripts/misc/kconfig/run.py new file mode 100644 index 000000000000..2689cb98d21b --- /dev/null +++ b/debian/scripts/misc/kconfig/run.py @@ -0,0 +1,365 @@ +# -*- mode: python -*- +# Manage Ubuntu kernel .config and annotations +# Copyright © 2022 Canonical Ltd. + +import sys +import os +import argparse +import json +from signal import signal, SIGPIPE, SIG_DFL + +try: + from argcomplete import autocomplete +except ModuleNotFoundError: + # Allow to run this program also when argcomplete is not available + def autocomplete(_unused): + pass + + +from kconfig.annotations import Annotation, KConfig # noqa: E402 Import not at top of file +from kconfig.utils import autodetect_annotations, arg_fail # noqa: E402 Import not at top of file +from kconfig.version import VERSION, ANNOTATIONS_FORMAT_VERSION # noqa: E402 Import not at top of file + + +SKIP_CONFIGS = ( + # CONFIG_VERSION_SIGNATURE is dynamically set during the build + "CONFIG_VERSION_SIGNATURE", + # Allow to use a different versions of toolchain tools + "CONFIG_GCC_VERSION", + "CONFIG_CC_VERSION_TEXT", + "CONFIG_AS_VERSION", + "CONFIG_LD_VERSION", + "CONFIG_LLD_VERSION", + "CONFIG_CLANG_VERSION", + "CONFIG_PAHOLE_VERSION", + "CONFIG_RUSTC_VERSION_TEXT", + "CONFIG_BINDGEN_VERSION_TEXT", +) + + +def make_parser(): + parser = argparse.ArgumentParser( + description="Manage Ubuntu kernel .config and annotations", + ) + parser.add_argument("--version", "-v", action="version", version=f"%(prog)s {VERSION}") + + parser.add_argument( + "--file", + "-f", + action="store", + help="Pass annotations or .config file to be parsed", + ) + parser.add_argument("--arch", "-a", action="store", help="Select architecture") + parser.add_argument("--flavour", "-l", action="store", help='Select flavour (default is "generic")') + parser.add_argument("--config", "-c", action="store", help="Select a specific config option") + parser.add_argument("--query", "-q", action="store_true", help="Query annotations") + parser.add_argument( + "--note", + "-n", + action="store", + help="Write a specific note to a config option in annotations", + ) + parser.add_argument( + "--autocomplete", + action="store_true", + help="Enable config bash autocomplete: `source <(annotations --autocomplete)`", + ) + parser.add_argument( + "--source", + "-t", + action="store_true", + help="Jump to a config definition in the kernel source code", + ) + parser.add_argument( + "--no-include", + action="store_true", + help="Do not process included annotations (stop at the main file)", + ) + + ga = parser.add_argument_group(title="Action").add_mutually_exclusive_group(required=False) + ga.add_argument( + "--write", + "-w", + action="store", + metavar="VALUE", + dest="value", + help="Set a specific config value in annotations (use 'null' to remove)", + ) + ga.add_argument( + "--export", + "-e", + action="store_true", + help="Convert annotations to .config format", + ) + ga.add_argument( + "--import", + "-i", + action="store", + metavar="FILE", + dest="import_file", + help="Import a full .config for a specific arch and flavour into annotations", + ) + ga.add_argument( + "--update", + "-u", + action="store", + metavar="FILE", + dest="update_file", + help="Import a partial .config into annotations (only resync configs specified in FILE)", + ) + ga.add_argument( + "--check", + "-k", + action="store", + metavar="FILE", + dest="check_file", + help="Validate kernel .config with annotations", + ) + return parser + + +_ARGPARSER = make_parser() + + +def export_result(data): + # Dump metadata / attributes first + out = '{\n "attributes": {\n' + for key, value in sorted(data["attributes"].items()): + out += f' "{key}": {json.dumps(value)},\n' + out = out.rstrip(",\n") + out += "\n }," + print(out) + + configs_with_note = {key: value for key, value in data["config"].items() if "note" in value} + configs_without_note = {key: value for key, value in data["config"].items() if "note" not in value} + + # Dump configs, sorted alphabetically, showing items with a note first + out = ' "config": {\n' + for key in sorted(configs_with_note) + sorted(configs_without_note): + policy = data["config"][key]["policy"] + if "note" in data["config"][key]: + note = data["config"][key]["note"] + out += f' "{key}": {{"policy": {json.dumps(policy)}, "note": {json.dumps(note)}}},\n' + else: + out += f' "{key}": {{"policy": {json.dumps(policy)}}},\n' + out = out.rstrip(",\n") + out += "\n }\n}" + print(out) + + +def print_result(config, data): + if data is not None and config is not None and config not in data: + data = {config: data} + print(json.dumps(data, sort_keys=True, indent=2)) + + +def do_query(args): + if args.arch is None and args.flavour is not None: + arg_fail(_ARGPARSER, "error: --flavour requires --arch") + a = Annotation(args.file, do_include=(not args.no_include)) + res = a.search_config(config=args.config, arch=args.arch, flavour=args.flavour) + # If no arguments are specified dump the whole annotations structure + if args.config is None and args.arch is None and args.flavour is None: + res = { + "attributes": { + "arch": a.arch, + "flavour": a.flavour, + "flavour_dep": a.flavour_dep, + "include": a.include, + "_version": ANNOTATIONS_FORMAT_VERSION, + }, + "config": res, + } + export_result(res) + else: + print_result(args.config, res) + + +def do_autocomplete(args): + a = Annotation(args.file) + res = (c.removeprefix("CONFIG_") for c in a.search_config()) + res_str = " ".join(res) + print(f'complete -W "{res_str}" annotations') + + +def do_source(args): + if args.config is None: + arg_fail(_ARGPARSER, "error: --source requires --config") + if not os.path.exists("tags"): + print("tags not found in the current directory, try: `make tags`") + sys.exit(1) + os.system(f"vim -t {args.config}") + + +def do_note(args): + if args.config is None: + arg_fail(_ARGPARSER, "error: --note requires --config") + + # Set the note in annotations + a = Annotation(args.file) + a.set(args.config, note=args.note) + + # Save back to annotations + a.save(args.file) + + # Query and print back the value + a = Annotation(args.file) + res = a.search_config(config=args.config) + print_result(args.config, res) + + +def do_write(args): + if args.config is None: + arg_fail(_ARGPARSER, "error: --write requires --config") + + # Set the value in annotations ('null' means remove) + a = Annotation(args.file) + if args.value == "null": + a.remove(args.config, arch=args.arch, flavour=args.flavour) + else: + a.set( + args.config, + arch=args.arch, + flavour=args.flavour, + value=args.value, + note=args.note, + ) + + # Save back to annotations + a.save(args.file) + + # Query and print back the value + a = Annotation(args.file) + res = a.search_config(config=args.config) + print_result(args.config, res) + + +def do_export(args): + if args.arch is None: + arg_fail(_ARGPARSER, "error: --export requires --arch") + a = Annotation(args.file) + conf = a.search_config(config=args.config, arch=args.arch, flavour=args.flavour) + if conf: + print(a.to_config(conf)) + + +def do_import(args): + if args.arch is None: + arg_fail(_ARGPARSER, "error: --arch is required with --import") + if args.flavour is None: + arg_fail(_ARGPARSER, "error: --flavour is required with --import") + if args.config is not None: + arg_fail(_ARGPARSER, "error: --config cannot be used with --import (try --update)") + + # Merge with the current annotations + a = Annotation(args.file) + c = KConfig(args.import_file) + a.update(c, arch=args.arch, flavour=args.flavour) + + # Save back to annotations + a.save(args.file) + + +def do_update(args): + if args.arch is None: + arg_fail(_ARGPARSER, "error: --arch is required with --update") + + # Merge with the current annotations + a = Annotation(args.file) + c = KConfig(args.update_file) + if args.config is None: + configs = list(set(c.config.keys()) - set(SKIP_CONFIGS)) + if configs: + a.update(c, arch=args.arch, flavour=args.flavour, configs=configs) + + # Save back to annotations + a.save(args.file) + + +def do_check(args): + # Determine arch and flavour + if args.arch is None: + arg_fail(_ARGPARSER, "error: --arch is required with --check") + + print(f"check-config: loading annotations from {args.file}") + total = good = ret = 0 + + # Load annotations settings + a = Annotation(args.file) + a_configs = a.search_config(arch=args.arch, flavour=args.flavour).keys() + + # Parse target .config + c = KConfig(args.check_file) + c_configs = c.config.keys() + + # Validate .config against annotations + for conf in sorted(a_configs | c_configs): + if conf in SKIP_CONFIGS: + continue + entry = a.search_config(config=conf, arch=args.arch, flavour=args.flavour) + expected = entry[conf] if entry else "-" + value = c.config[conf] if conf in c.config else "-" + if value != expected: + policy = a.config[conf] if conf in a.config else "undefined" + if "policy" in policy: + policy = f"policy<{policy['policy']}>" + print(f"check-config: {conf} changed from {expected} to {value}: {policy})") + ret = 1 + else: + good += 1 + total += 1 + + num = total - good + if ret: + if os.path.exists(".git"): + print(f"check-config: {num} config options have been changed, review them with `git diff`") + else: + print(f"check-config: {num} config options have changed") + else: + print("check-config: all good") + sys.exit(ret) + + +def main(): + # Prevent broken pipe errors when showing output in pipe to other tools + # (less for example) + signal(SIGPIPE, SIG_DFL) + + # Main annotations program + autocomplete(_ARGPARSER) + args = _ARGPARSER.parse_args() + + if args.file is None: + args.file = autodetect_annotations() + if args.file is None: + arg_fail( + _ARGPARSER, + "error: could not determine DEBDIR, try using: --file/-f", + show_usage=False, + ) + + if args.config and not args.config.startswith("CONFIG_"): + args.config = "CONFIG_" + args.config + + if args.value: + do_write(args) + elif args.note: + do_note(args) + elif args.export: + do_export(args) + elif args.import_file: + do_import(args) + elif args.update_file: + do_update(args) + elif args.check_file: + do_check(args) + elif args.autocomplete: + do_autocomplete(args) + elif args.source: + do_source(args) + else: + do_query(args) + + +if __name__ == "__main__": + main() diff --git a/debian/scripts/misc/kconfig/utils.py b/debian/scripts/misc/kconfig/utils.py new file mode 100644 index 000000000000..f4ced6b33f4a --- /dev/null +++ b/debian/scripts/misc/kconfig/utils.py @@ -0,0 +1,20 @@ +# -*- mode: python -*- +# Misc helpers for Kconfig and annotations +# Copyright © 2023 Canonical Ltd. + +import sys + + +def autodetect_annotations(): + try: + with open("debian/debian.env", "rt", encoding="utf-8") as fd: + return fd.read().rstrip().split("=")[1] + "/config/annotations" + except (FileNotFoundError, IndexError): + return None + + +def arg_fail(parser, message, show_usage=True): + print(message) + if show_usage: + parser.print_usage() + sys.exit(1) diff --git a/debian/scripts/misc/kconfig/version.py b/debian/scripts/misc/kconfig/version.py new file mode 100644 index 000000000000..833ffa34654b --- /dev/null +++ b/debian/scripts/misc/kconfig/version.py @@ -0,0 +1,10 @@ +# -*- mode: python -*- +# version of annotations module +# Copyright © 2022 Canonical Ltd. + +VERSION = "0.1" + +ANNOTATIONS_FORMAT_VERSION = 5 + +if __name__ == "__main__": + print(VERSION)