diff --git a/debian/scripts/misc/kconfig/annotations.py b/debian/scripts/misc/kconfig/annotations.py
index 20e7451a0ae1..84dd4d26e6af 100644
--- a/debian/scripts/misc/kconfig/annotations.py
+++ b/debian/scripts/misc/kconfig/annotations.py
@@ -236,6 +236,17 @@ class Annotation(Config):
                         for flavour in arch_flavours:
                             del self.config[conf]['policy'][flavour]
                         self.config[conf]['policy'][arch] = value
+        # After the first round of compaction we may end up having configs that
+        # are undefined across all arches, so do another round of compaction to
+        # drop these settings that are not needed anymore.
+        for conf in self.config.copy():
+            # Remove configs that are all undefined across all arches/flavours
+            if 'policy' in self.config[conf]:
+                if list(set(self.config[conf]['policy'].values())) == ['-']:
+                    self.config[conf]['policy'] = {}
+            # Drop empty rules
+            if not self.config[conf]['policy']:
+                del self.config[conf]
 
     def save(self, fname: str):
         """ Save annotations data to the annotation file """