UBUNTU: [Packaging] introduce annotations script
Add a script to manage annotations and .config. Signed-off-by: Andrea Righi <andrea.righi@canonical.com>
This commit is contained in:
Andrea Righi
committed by
Paolo Pisati
Paolo Pisati
parent
5feb0881e2
commit
b9c9629053
+163
@@ -0,0 +1,163 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- mode: python -*-
|
||||
# Manage Ubuntu kernel .config and annotations
|
||||
# Copyright © 2022 Canonical Ltd.
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import json
|
||||
from kconfig.annotations import Annotation, KConfig
|
||||
|
||||
VERSION = '0.1'
|
||||
|
||||
SKIP_CONFIGS = (
|
||||
# CONFIG_VERSION_SIGNATURE is dynamically set during the build
|
||||
'CONFIG_VERSION_SIGNATURE',
|
||||
# Allow to use a different version of gcc
|
||||
'CONFIG_CC_VERSION_TEXT',
|
||||
)
|
||||
|
||||
def make_parser():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Manage Ubuntu kernel .config and annotations',
|
||||
)
|
||||
parser.add_argument('--version', '-v', action='version', version=f'%(prog)s {VERSION}')
|
||||
|
||||
parser.add_argument('--file', '-f', action='store',
|
||||
help='Pass annotations or .config file to be parsed')
|
||||
parser.add_argument('--arch', '-a', action='store',
|
||||
help='Select architecture')
|
||||
parser.add_argument('--flavour', '-l', action='store',
|
||||
help='Select flavour (default is "generic")')
|
||||
parser.add_argument('--config', '-c', action='store',
|
||||
help='Select a specific config option')
|
||||
|
||||
ga = parser.add_argument_group(title='Action').add_mutually_exclusive_group(required=True)
|
||||
ga.add_argument('--query', '-q', action='store_true',
|
||||
help='Query annotations')
|
||||
ga.add_argument('--export', '-e', action='store_true',
|
||||
help='Convert annotations to .config format')
|
||||
ga.add_argument('--import', '-i', action='store',
|
||||
metavar="FILE", dest='import_file',
|
||||
help='Import a full .config for a specific arch and flavour into annotations')
|
||||
ga.add_argument('--update', '-u', action='store',
|
||||
metavar="FILE", dest='update_file',
|
||||
help='Import a partial .config into annotations (only resync configs specified in FILE)')
|
||||
ga.add_argument('--check', '-k', action='store',
|
||||
metavar="FILE", dest='check_file',
|
||||
help='Validate kernel .config with annotations')
|
||||
return parser
|
||||
|
||||
_ARGPARSER = make_parser()
|
||||
|
||||
def arg_fail(message):
|
||||
print(message)
|
||||
_ARGPARSER.print_usage()
|
||||
exit(1)
|
||||
|
||||
def do_query(args):
|
||||
a = Annotation(args.file)
|
||||
res = a.search_config(config=args.config, arch=args.arch, flavour=args.flavour)
|
||||
print(json.dumps(res, indent=4))
|
||||
|
||||
def do_export(args):
|
||||
if args.arch is None:
|
||||
arg_fail('error: --export requires --arch')
|
||||
a = Annotation(args.file)
|
||||
conf = a.search_config(config=args.config, arch=args.arch, flavour=args.flavour)
|
||||
if conf:
|
||||
print(a.to_config(conf))
|
||||
|
||||
def do_import(args):
|
||||
if args.arch is None:
|
||||
arg_fail('error: --arch and --flavour are required with --import')
|
||||
if args.flavour is None:
|
||||
arg_fail('error: --arch and --flavour are required with --import')
|
||||
if args.config is not None:
|
||||
arg_fail('error: --config cannot be used with --import (try --update)')
|
||||
|
||||
# Merge with the current annotations
|
||||
a = Annotation(args.file)
|
||||
c = KConfig(args.import_file)
|
||||
a.update(c, arch=args.arch, flavour=args.flavour)
|
||||
|
||||
# Save back to annotations
|
||||
a.save(args.file)
|
||||
|
||||
def do_update(args):
|
||||
if args.arch is None:
|
||||
arg_fail('error: --arch is required with --update')
|
||||
|
||||
# Merge with the current annotations
|
||||
a = Annotation(args.file)
|
||||
c = KConfig(args.update_file)
|
||||
if args.config is None:
|
||||
configs = list(set(c.config.keys()) - set(SKIP_CONFIGS))
|
||||
a.update(c, arch=args.arch, flavour=args.flavour, configs=configs)
|
||||
|
||||
# Save back to annotations
|
||||
a.save(args.file)
|
||||
|
||||
def do_check(args):
|
||||
# Determine arch and flavour
|
||||
if args.arch is None:
|
||||
arg_fail('error: --arch is required with --check')
|
||||
|
||||
print(f"check-config: loading annotations from {args.file}")
|
||||
total = good = ret = 0
|
||||
|
||||
# Load annotations settings
|
||||
a = Annotation(args.file)
|
||||
a_configs = a.search_config(arch=args.arch, flavour=args.flavour).keys()
|
||||
|
||||
# Parse target .config
|
||||
c = KConfig(args.check_file)
|
||||
c_configs = c.config.keys()
|
||||
|
||||
# Validate .config against annotations
|
||||
for conf in a_configs | c_configs:
|
||||
if conf in SKIP_CONFIGS:
|
||||
continue
|
||||
entry = a.search_config(config=conf, arch=args.arch, flavour=args.flavour)
|
||||
expected = entry[conf] if entry else '-'
|
||||
value = c.config[conf] if conf in c.config else '-'
|
||||
if value != expected:
|
||||
policy = a.config[conf] if conf in a.config else 'undefined'
|
||||
if 'policy' in policy:
|
||||
policy = f"policy<{policy['policy']}>"
|
||||
print(f"check-config: FAIL: ({value} != {expected}): {conf} {policy})")
|
||||
ret = 1
|
||||
else:
|
||||
good += 1
|
||||
total += 1
|
||||
|
||||
print(f"check-config: {good}/{total} checks passed -- exit {ret}")
|
||||
exit(ret)
|
||||
|
||||
def autodetect_annotations(args):
|
||||
if args.file:
|
||||
return
|
||||
# If --file/-f isn't specified try to automatically determine the right
|
||||
# location of the annotations file looking at debian/debian.env.
|
||||
try:
|
||||
with open('debian/debian.env', 'rt') as fd:
|
||||
args.file = fd.read().rstrip().split('=')[1] + '/config/annotations'
|
||||
except:
|
||||
arg_fail('error: could not determine DEBDIR, try using: --file/-f')
|
||||
|
||||
def main():
|
||||
args = _ARGPARSER.parse_args()
|
||||
autodetect_annotations(args)
|
||||
if args.query:
|
||||
do_query(args)
|
||||
elif args.export:
|
||||
do_export(args)
|
||||
elif args.import_file:
|
||||
do_import(args)
|
||||
elif args.update_file:
|
||||
do_update(args)
|
||||
elif args.check_file:
|
||||
do_check(args)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+240
@@ -0,0 +1,240 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- mode: python -*-
|
||||
# python module to manage Ubuntu kernel .config and annotations
|
||||
# Copyright © 2022 Canonical Ltd.
|
||||
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import tempfile
|
||||
from ast import literal_eval
|
||||
from os.path import dirname, abspath
|
||||
|
||||
class Config(object):
|
||||
def __init__(self, fname: str, arch: str = None, flavour: str = None):
|
||||
"""
|
||||
Basic configuration file object
|
||||
"""
|
||||
self.fname = fname
|
||||
raw_data = self._load(fname)
|
||||
self.config = self._parse(raw_data)
|
||||
|
||||
def _load(self, fname: str) -> str:
|
||||
with open(fname, 'rt') as fd:
|
||||
data = fd.read()
|
||||
return data.rstrip()
|
||||
|
||||
def __str__(self):
|
||||
""" Return a JSON representation of the config """
|
||||
return json.dumps(self.config, indent=4)
|
||||
|
||||
class KConfig(Config):
|
||||
"""
|
||||
Parse a .config file, individual config options can be accessed via
|
||||
.config[<CONFIG_OPTION>]
|
||||
"""
|
||||
def _parse(self, data: str) -> dict:
|
||||
config = {}
|
||||
for line in data.splitlines():
|
||||
m = re.match(r'^# (CONFIG_.*) is not set$', line)
|
||||
if m:
|
||||
config[m.group(1)] = literal_eval("'n'")
|
||||
continue
|
||||
m = re.match(r'^(CONFIG_[A-Za-z0-9_]+)=(.*)$', line)
|
||||
if m:
|
||||
config[m.group(1)] = literal_eval("'" + m.group(2) + "'")
|
||||
continue
|
||||
return config
|
||||
|
||||
class Annotation(Config):
|
||||
"""
|
||||
Parse annotations file, individual config options can be accessed via
|
||||
.config[<CONFIG_OPTION>]
|
||||
"""
|
||||
def _parse(self, data: str) -> dict:
|
||||
# Parse header
|
||||
self.header = ''
|
||||
for line in data.splitlines():
|
||||
if re.match(r'^#.*', line):
|
||||
m = re.match(r'^# ARCH: (.*)', line)
|
||||
if m:
|
||||
self.arch = list(m.group(1).split(' '))
|
||||
m = re.match(r'^# FLAVOUR: (.*)', line)
|
||||
if m:
|
||||
self.flavour = list(m.group(1).split(' '))
|
||||
self.header += line + "\n"
|
||||
else:
|
||||
break
|
||||
|
||||
# Skip comments
|
||||
data = re.sub(r'(?m)^\s*#.*\n?', '', data)
|
||||
|
||||
# Handle includes (recursively)
|
||||
self.include = []
|
||||
expand_data = ''
|
||||
for line in data.splitlines():
|
||||
m = re.match(r'^include\s+"?([^"]*)"?', line)
|
||||
if m:
|
||||
self.include.append(m.group(1))
|
||||
include_fname = dirname(abspath(self.fname)) + '/' + m.group(1)
|
||||
include_data = self._load(include_fname)
|
||||
expand_data += include_data + '\n'
|
||||
else:
|
||||
expand_data += line + '\n'
|
||||
|
||||
# Skip empty, non-policy and non-note lines
|
||||
data = "\n".join([l.rstrip() for l in expand_data.splitlines()
|
||||
if l.strip() and (re.match('.* policy<', l) or re.match('.* note<', l))])
|
||||
|
||||
# Convert multiple spaces to single space to simplifly parsing
|
||||
data = re.sub(r' *', ' ', data)
|
||||
|
||||
# Parse config/note statements
|
||||
config = {}
|
||||
for line in data.splitlines():
|
||||
try:
|
||||
conf = line.split(' ')[0]
|
||||
if conf in config:
|
||||
entry = config[conf]
|
||||
else:
|
||||
entry = {}
|
||||
m = re.match(r'.*policy<(.*)>', line)
|
||||
if m:
|
||||
entry['policy'] = literal_eval(m.group(1))
|
||||
m = re.match(r'.*note<(.*?)>', line)
|
||||
if m:
|
||||
entry['note'] = "'" + m.group(1).replace("'", '') + "'"
|
||||
if entry:
|
||||
config[conf] = entry
|
||||
except Exception as e:
|
||||
raise Exception(str(e) + f', line = {line}')
|
||||
return config
|
||||
|
||||
def update(self, c: KConfig, arch: str, flavour: str = None, configs: list = []):
|
||||
""" Merge configs from a Kconfig object into Annotation object """
|
||||
|
||||
# Determine if we need to import all configs or a single config
|
||||
if not configs:
|
||||
configs = c.config.keys()
|
||||
configs |= self.search_config(arch=arch, flavour=flavour).keys()
|
||||
|
||||
# Import configs from the Kconfig object into Annotations
|
||||
if flavour is not None:
|
||||
flavour = arch + f'-{flavour}'
|
||||
else:
|
||||
flavour = arch
|
||||
for conf in configs:
|
||||
if conf in c.config:
|
||||
val = c.config[conf]
|
||||
else:
|
||||
val = '-'
|
||||
if conf in self.config:
|
||||
if 'policy' in self.config[conf]:
|
||||
self.config[conf]['policy'][flavour] = val
|
||||
else:
|
||||
self.config[conf]['policy'] = {flavour: val}
|
||||
else:
|
||||
self.config[conf] = {'policy': {flavour: val}}
|
||||
|
||||
def _compact(self):
|
||||
# Try to remove redundant settings: if the config value of a flavour is
|
||||
# the same as the one of the main arch simply drop it.
|
||||
for conf in self.config:
|
||||
if 'policy' not in self.config[conf]:
|
||||
continue
|
||||
for flavour in self.flavour:
|
||||
if flavour not in self.config[conf]['policy']:
|
||||
continue
|
||||
m = re.match(r'^(.*?)-(.*)$', flavour)
|
||||
if not m:
|
||||
continue
|
||||
arch = m.group(1)
|
||||
if arch not in self.config[conf]['policy']:
|
||||
continue
|
||||
if self.config[conf]['policy'][flavour] == self.config[conf]['policy'][arch]:
|
||||
del self.config[conf]['policy'][flavour]
|
||||
|
||||
def save(self, fname: str):
|
||||
""" Save annotations data to the annotation file """
|
||||
# Compact annotations structure
|
||||
self._compact()
|
||||
|
||||
# Save annotations to disk
|
||||
with tempfile.NamedTemporaryFile(mode='w+t', delete=False) as tmp:
|
||||
# Write header
|
||||
tmp.write(self.header + '\n')
|
||||
|
||||
# Write includes
|
||||
for i in self.include:
|
||||
tmp.write(f'include "{i}"\n')
|
||||
if self.include:
|
||||
tmp.write("\n")
|
||||
|
||||
# Write config annotations and notes
|
||||
tmp.flush()
|
||||
shutil.copy(tmp.name, fname)
|
||||
tmp_a = Annotation(fname)
|
||||
|
||||
# Only save local differences (preserve includes)
|
||||
for conf in sorted(self.config):
|
||||
old_val = tmp_a.config[conf] if conf in tmp_a.config else None
|
||||
new_val = self.config[conf]
|
||||
if old_val != new_val:
|
||||
if 'policy' in self.config[conf]:
|
||||
val = dict(sorted(self.config[conf]['policy'].items()))
|
||||
line = f"{conf : <47} policy<{val}>"
|
||||
tmp.write(line + "\n")
|
||||
if 'note' in self.config[conf]:
|
||||
val = self.config[conf]['note']
|
||||
line = f"{conf : <47} note<{val}>"
|
||||
tmp.write(line + "\n\n")
|
||||
|
||||
# Replace annotations with the updated version
|
||||
tmp.flush()
|
||||
shutil.move(tmp.name, fname)
|
||||
|
||||
def search_config(self, config: str = None, arch: str = None, flavour: str = None) -> dict:
|
||||
""" Return config value of a specific config option or architecture """
|
||||
if flavour is None:
|
||||
flavour = 'generic'
|
||||
flavour = f'{arch}-{flavour}'
|
||||
if config is None and arch is None:
|
||||
# Get all config options for all architectures
|
||||
return self.config
|
||||
elif config is None and arch is not None:
|
||||
# Get config options of a specific architecture
|
||||
ret = {}
|
||||
for c in self.config:
|
||||
if not 'policy' in self.config[c]:
|
||||
continue
|
||||
if flavour in self.config[c]['policy']:
|
||||
ret[c] = self.config[c]['policy'][flavour]
|
||||
elif arch in self.config[c]['policy']:
|
||||
ret[c] = self.config[c]['policy'][arch]
|
||||
return ret
|
||||
elif config is not None and arch is None:
|
||||
# Get a specific config option for all architectures
|
||||
return self.config[config] if config in self.config else None
|
||||
elif config is not None and arch is not None:
|
||||
# Get a specific config option for a specific architecture
|
||||
if config in self.config:
|
||||
if 'policy' in self.config[config]:
|
||||
if flavour in self.config[config]['policy']:
|
||||
return {config: self.config[config]['policy'][flavour]}
|
||||
elif arch in self.config[config]['policy']:
|
||||
return {config: self.config[config]['policy'][arch]}
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def to_config(data: dict) -> str:
|
||||
""" Convert annotations data to .config format """
|
||||
s = ''
|
||||
for c in data:
|
||||
v = data[c]
|
||||
if v == 'n':
|
||||
s += f"# {c} is not set\n"
|
||||
elif v == '-':
|
||||
pass
|
||||
else:
|
||||
s += f"{c}={v}\n"
|
||||
return s.rstrip()
|
||||
Reference in New Issue
Block a user