From 2185242faddd12a1ba1060be5caf584fe5aba93a Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 7 Jan 2023 18:18:17 +0900 Subject: [PATCH 1/5] kbuild: remove sed commands after rustc rules rustc may put comments in dep-info, so sed is used to drop them before passing it to fixdep. Now that fixdep can remove comments, Makefiles do not need to run sed. Signed-off-by: Masahiro Yamada Reviewed-by: Miguel Ojeda Tested-by: Miguel Ojeda Reviewed-by: Vincenzo Palazzo --- rust/Makefile | 6 ++---- scripts/Makefile.build | 18 ++++-------------- scripts/Makefile.host | 3 +-- 3 files changed, 7 insertions(+), 20 deletions(-) diff --git a/rust/Makefile b/rust/Makefile index 865afb87bc9b..f403b79cae5a 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -333,8 +333,7 @@ quiet_cmd_rustc_procmacro = $(RUSTC_OR_CLIPPY_QUIET) P $@ $(RUSTC_OR_CLIPPY) $(rust_common_flags) \ --emit=dep-info=$(depfile) --emit=link=$@ --extern proc_macro \ --crate-type proc-macro \ - --crate-name $(patsubst lib%.so,%,$(notdir $@)) $<; \ - sed -i '/^\#/d' $(depfile) + --crate-name $(patsubst lib%.so,%,$(notdir $@)) $< # Procedural macros can only be used with the `rustc` that compiled it. # Therefore, to get `libmacros.so` automatically recompiled when the compiler @@ -350,8 +349,7 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L --emit=dep-info=$(depfile) --emit=obj=$@ \ --emit=metadata=$(dir $@)$(patsubst %.o,lib%.rmeta,$(notdir $@)) \ --crate-type rlib -L$(objtree)/$(obj) \ - --crate-name $(patsubst %.o,%,$(notdir $@)) $<; \ - sed -i '/^\#/d' $(depfile) \ + --crate-name $(patsubst %.o,%,$(notdir $@)) $< \ $(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@) rust-analyzer: diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 40de20246e50..76323201232a 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -289,9 +289,6 @@ rust_common_cmd = \ --crate-name $(basename $(notdir $@)) \ --emit=dep-info=$(depfile) -rust_handle_depfile = \ - sed -i '/^\#/d' $(depfile) - # `--emit=obj`, `--emit=asm` and `--emit=llvm-ir` imply a single codegen unit # will be used. We explicitly request `-Ccodegen-units=1` in any case, and # the compiler shows a warning if it is not 1. However, if we ever stop @@ -301,9 +298,7 @@ rust_handle_depfile = \ # would not match each other. quiet_cmd_rustc_o_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ - cmd_rustc_o_rs = \ - $(rust_common_cmd) --emit=obj=$@ $<; \ - $(rust_handle_depfile) + cmd_rustc_o_rs = $(rust_common_cmd) --emit=obj=$@ $< $(obj)/%.o: $(src)/%.rs FORCE $(call if_changed_dep,rustc_o_rs) @@ -311,24 +306,19 @@ $(obj)/%.o: $(src)/%.rs FORCE quiet_cmd_rustc_rsi_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ cmd_rustc_rsi_rs = \ $(rust_common_cmd) -Zunpretty=expanded $< >$@; \ - command -v $(RUSTFMT) >/dev/null && $(RUSTFMT) $@; \ - $(rust_handle_depfile) + command -v $(RUSTFMT) >/dev/null && $(RUSTFMT) $@ $(obj)/%.rsi: $(src)/%.rs FORCE $(call if_changed_dep,rustc_rsi_rs) quiet_cmd_rustc_s_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ - cmd_rustc_s_rs = \ - $(rust_common_cmd) --emit=asm=$@ $<; \ - $(rust_handle_depfile) + cmd_rustc_s_rs = $(rust_common_cmd) --emit=asm=$@ $< $(obj)/%.s: $(src)/%.rs FORCE $(call if_changed_dep,rustc_s_rs) quiet_cmd_rustc_ll_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ - cmd_rustc_ll_rs = \ - $(rust_common_cmd) --emit=llvm-ir=$@ $<; \ - $(rust_handle_depfile) + cmd_rustc_ll_rs = $(rust_common_cmd) --emit=llvm-ir=$@ $< $(obj)/%.ll: $(src)/%.rs FORCE $(call if_changed_dep,rustc_ll_rs) diff --git a/scripts/Makefile.host b/scripts/Makefile.host index d812241144d4..7aea9005e497 100644 --- a/scripts/Makefile.host +++ b/scripts/Makefile.host @@ -148,8 +148,7 @@ $(host-cxxobjs): $(obj)/%.o: $(src)/%.cc FORCE # host-rust -> Executable quiet_cmd_host-rust = HOSTRUSTC $@ cmd_host-rust = \ - $(HOSTRUSTC) $(hostrust_flags) --emit=link=$@ $<; \ - sed -i '/^\#/d' $(depfile) + $(HOSTRUSTC) $(hostrust_flags) --emit=link=$@ $< $(host-rust): $(obj)/%: $(src)/%.rs FORCE $(call if_changed_dep,host-rust) From 871d657385466d380133e457fc86672bedf14cd9 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 7 Jan 2023 18:18:18 +0900 Subject: [PATCH 2/5] fixdep: refactor hash table lookup Change the hash table code so it will be easier to add the second table. Signed-off-by: Masahiro Yamada Reviewed-by: Miguel Ojeda Tested-by: Miguel Ojeda --- scripts/basic/fixdep.c | 47 ++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/scripts/basic/fixdep.c b/scripts/basic/fixdep.c index f5a51770eb74..74f90a0deeb9 100644 --- a/scripts/basic/fixdep.c +++ b/scripts/basic/fixdep.c @@ -113,7 +113,7 @@ struct item { }; #define HASHSZ 256 -static struct item *hashtab[HASHSZ]; +static struct item *config_hashtab[HASHSZ]; static unsigned int strhash(const char *str, unsigned int sz) { @@ -125,25 +125,11 @@ static unsigned int strhash(const char *str, unsigned int sz) return hash; } -/* - * Lookup a value in the configuration string. - */ -static int is_defined_config(const char *name, int len, unsigned int hash) -{ - struct item *aux; - - for (aux = hashtab[hash % HASHSZ]; aux; aux = aux->next) { - if (aux->hash == hash && aux->len == len && - memcmp(aux->name, name, len) == 0) - return 1; - } - return 0; -} - /* * Add a new value to the configuration string. */ -static void define_config(const char *name, int len, unsigned int hash) +static void add_to_hashtable(const char *name, int len, unsigned int hash, + struct item *hashtab[]) { struct item *aux = malloc(sizeof(*aux) + len); @@ -158,17 +144,34 @@ static void define_config(const char *name, int len, unsigned int hash) hashtab[hash % HASHSZ] = aux; } +/* + * Lookup a string in the hash table. If found, just return true. + * If not, add it to the hashtable and return false. + */ +static bool in_hashtable(const char *name, int len, struct item *hashtab[]) +{ + struct item *aux; + unsigned int hash = strhash(name, len); + + for (aux = hashtab[hash % HASHSZ]; aux; aux = aux->next) { + if (aux->hash == hash && aux->len == len && + memcmp(aux->name, name, len) == 0) + return true; + } + + add_to_hashtable(name, len, hash, hashtab); + + return false; +} + /* * Record the use of a CONFIG_* word. */ static void use_config(const char *m, int slen) { - unsigned int hash = strhash(m, slen); + if (in_hashtable(m, slen, config_hashtab)) + return; - if (is_defined_config(m, slen, hash)) - return; - - define_config(m, slen, hash); /* Print out a dependency path from a symbol name. */ printf(" $(wildcard include/config/%.*s) \\\n", slen, m); } From faa91c472be8ffa3121c9db803d3e529d48e246a Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 7 Jan 2023 18:18:19 +0900 Subject: [PATCH 3/5] fixdep: avoid parsing the same file over again The dep files (*.d files) emitted by C compilers usually contain the deduplicated list of included files. One exceptional case is when a header is included by the -include command line option, and also by #include directive. For example, the top Makefile adds the command line option, "-include $(srctree)/include/linux/kconfig.h". You do not need to include in every source file. In fact, include/linux/kconfig.h is listed twice in many .*.cmd files due to include/linux/xarray.h having "#include ". I did not fix that since it is a small redundancy. However, this is more annoying for rustc. rustc emits the dependency for each emission type. For example, cmd_rustc_library emits dep-info, obj, and metadata. So, the emitted *.d file contains the dependency for those 3 targets, which makes fixdep parse the same file 3 times. $ grep rust/alloc/raw_vec.rs rust/.alloc.o.cmd rust/alloc/raw_vec.rs \ rust/alloc/raw_vec.rs \ rust/alloc/raw_vec.rs \ To skip the second parsing, this commit adds a hash table for parsed files, just like we did for CONFIG options. Signed-off-by: Masahiro Yamada Acked-by: Miguel Ojeda Tested-by: Miguel Ojeda Reviewed-by: Vincenzo Palazzo --- scripts/basic/fixdep.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/basic/fixdep.c b/scripts/basic/fixdep.c index 74f90a0deeb9..e22e689de61e 100644 --- a/scripts/basic/fixdep.c +++ b/scripts/basic/fixdep.c @@ -113,7 +113,7 @@ struct item { }; #define HASHSZ 256 -static struct item *config_hashtab[HASHSZ]; +static struct item *config_hashtab[HASHSZ], *file_hashtab[HASHSZ]; static unsigned int strhash(const char *str, unsigned int sz) { @@ -365,6 +365,10 @@ static void parse_dep_file(char *p, const char *target) * name, which will be the original one, and ignore any * other source names, which will be intermediate * temporary files. + * + * rustc emits the same dependency list for each + * emission type. It is enough to list the source name + * just once. */ if (!saw_any_target) { saw_any_target = true; @@ -372,7 +376,8 @@ static void parse_dep_file(char *p, const char *target) printf("deps_%s := \\\n", target); need_parse = true; } - } else if (!is_ignored_file(p, q - p)) { + } else if (!is_ignored_file(p, q - p) && + !in_hashtable(p, q - p, file_hashtab)) { printf(" %s \\\n", p); need_parse = true; } From 93c656de8da8b53317bd88ecb7cce592310995d8 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 7 Jan 2023 18:18:20 +0900 Subject: [PATCH 4/5] fixdep: do not parse *.rlib, *.rmeta, *.so fixdep is designed only for parsing text files. read_file() appends a terminating null byte ('\0') and parse_config_file() calls strstr() to search for CONFIG options. rustc outputs *.rlib, *.rmeta, *.so to dep-info. fixdep needs them in the dependency, but there is no point in parsing such binary files. Signed-off-by: Masahiro Yamada Reviewed-by: Miguel Ojeda Tested-by: Miguel Ojeda --- scripts/basic/fixdep.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/basic/fixdep.c b/scripts/basic/fixdep.c index e22e689de61e..fa562806c2be 100644 --- a/scripts/basic/fixdep.c +++ b/scripts/basic/fixdep.c @@ -250,6 +250,15 @@ static int is_ignored_file(const char *s, int len) str_ends_with(s, len, "include/generated/autoksyms.h"); } +/* Do not parse these files */ +static int is_no_parse_file(const char *s, int len) +{ + /* rustc may list binary files in dep-info */ + return str_ends_with(s, len, ".rlib") || + str_ends_with(s, len, ".rmeta") || + str_ends_with(s, len, ".so"); +} + /* * Important: The below generated source_foo.o and deps_foo.o variable * assignments are parsed not only by make, but also by the rather simple @@ -382,7 +391,7 @@ static void parse_dep_file(char *p, const char *target) need_parse = true; } - if (need_parse) { + if (need_parse && !is_no_parse_file(p, q - p)) { void *buf; buf = read_file(p); From c83b16cefd78f55071840e1159ead9fe62747769 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 7 Jan 2023 18:45:45 +0900 Subject: [PATCH 5/5] kbuild: rust: move rust/target.json to scripts/ scripts/ is a better place to generate files used treewide. With target.json moved to scripts/, you do not need to add target.json to no-clean-files or MRPROPER_FILES. 'make clean' does not visit scripts/, but 'make mrproper' does. Signed-off-by: Masahiro Yamada Reviewed-by: Miguel Ojeda Tested-by: Miguel Ojeda --- Makefile | 4 ++-- rust/.gitignore | 1 - rust/Makefile | 10 +--------- scripts/.gitignore | 1 + scripts/Makefile | 8 +++++++- scripts/remove-stale-files | 2 ++ 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index b9d1411ef25b..ef6d593634cf 100644 --- a/Makefile +++ b/Makefile @@ -569,7 +569,7 @@ KBUILD_CFLAGS := -Wall -Wundef -Werror=strict-prototypes -Wno-trigraphs \ -std=gnu11 KBUILD_CPPFLAGS := -D__KERNEL__ KBUILD_RUSTFLAGS := $(rust_common_flags) \ - --target=$(objtree)/rust/target.json \ + --target=$(objtree)/scripts/target.json \ -Cpanic=abort -Cembed-bitcode=n -Clto=n \ -Cforce-unwind-tables=n -Ccodegen-units=1 \ -Csymbol-mangling-version=v0 \ @@ -1606,7 +1606,7 @@ MRPROPER_FILES += include/config include/generated \ certs/x509.genkey \ vmlinux-gdb.py \ *.spec \ - rust/target.json rust/libmacros.so + rust/libmacros.so # clean - Delete most, but leave enough to build external modules # diff --git a/rust/.gitignore b/rust/.gitignore index 9bd1af8e05a1..168cb26a31b9 100644 --- a/rust/.gitignore +++ b/rust/.gitignore @@ -1,6 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -target.json bindings_generated.rs bindings_helpers_generated.rs exports_*_generated.h diff --git a/rust/Makefile b/rust/Makefile index f403b79cae5a..829f4adfd85c 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -1,8 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -always-$(CONFIG_RUST) += target.json -no-clean-files += target.json - obj-$(CONFIG_RUST) += core.o compiler_builtins.o always-$(CONFIG_RUST) += exports_core_generated.h @@ -231,11 +228,6 @@ rusttest-kernel: $(src)/kernel/lib.rs rusttest-prepare \ $(call if_changed,rustc_test) $(call if_changed,rustc_test_library) -filechk_rust_target = $(objtree)/scripts/generate_rust_target < $< - -$(obj)/target.json: $(objtree)/include/config/auto.conf FORCE - $(call filechk,rust_target) - ifdef CONFIG_CC_IS_CLANG bindgen_c_flags = $(c_flags) else @@ -359,7 +351,7 @@ rust-analyzer: $(obj)/core.o: private skip_clippy = 1 $(obj)/core.o: private skip_flags = -Dunreachable_pub $(obj)/core.o: private rustc_target_flags = $(core-cfgs) -$(obj)/core.o: $(RUST_LIB_SRC)/core/src/lib.rs $(obj)/target.json FORCE +$(obj)/core.o: $(RUST_LIB_SRC)/core/src/lib.rs scripts/target.json FORCE $(call if_changed_dep,rustc_library) $(obj)/compiler_builtins.o: private rustc_objcopy = -w -W '__*' diff --git a/scripts/.gitignore b/scripts/.gitignore index b7aec8eb1bd4..11bf3c075fb6 100644 --- a/scripts/.gitignore +++ b/scripts/.gitignore @@ -8,4 +8,5 @@ /recordmcount /sign-file /sorttable +/target.json /unifdef diff --git a/scripts/Makefile b/scripts/Makefile index 1575af84d557..0e0ae3c06ed7 100644 --- a/scripts/Makefile +++ b/scripts/Makefile @@ -10,8 +10,14 @@ hostprogs-always-$(CONFIG_BUILDTIME_TABLE_SORT) += sorttable hostprogs-always-$(CONFIG_ASN1) += asn1_compiler hostprogs-always-$(CONFIG_MODULE_SIG_FORMAT) += sign-file hostprogs-always-$(CONFIG_SYSTEM_EXTRA_CERTIFICATE) += insert-sys-cert -hostprogs-always-$(CONFIG_RUST) += generate_rust_target +always-$(CONFIG_RUST) += target.json +filechk_rust_target = $< < include/config/auto.conf + +$(obj)/target.json: scripts/generate_rust_target include/config/auto.conf FORCE + $(call filechk,rust_target) + +hostprogs += generate_rust_target generate_rust_target-rust := y HOSTCFLAGS_sorttable.o = -I$(srctree)/tools/include diff --git a/scripts/remove-stale-files b/scripts/remove-stale-files index cdbdde89a271..c71bf2f68360 100755 --- a/scripts/remove-stale-files +++ b/scripts/remove-stale-files @@ -27,3 +27,5 @@ rm -f arch/x86/purgatory/kexec-purgatory.c rm -f scripts/extract-cert rm -f scripts/kconfig/[gmnq]conf-cfg + +rm -f rust/target.json