ANDROID: kasan: flush dcache after tag writes

Certain H/W need to flush the dcache for cache coherency
after writing the tag. This is activated when 'kasan_inval_dcache'
is declared in the kernel parameter.

Bug: 382390925

Change-Id: I46ae3d5074a6dfe3a42f81a24d1fccf0dd6db64f
Signed-off-by: Hyesoo Yu <hyesoo.yu@samsung.com>

mm/kasan/hw_tags.c

@@ -83,6 +83,12 @@ unsigned int kasan_page_alloc_sample_order = PAGE_ALLOC_SAMPLE_ORDER_DEFAULT;
 
 DEFINE_PER_CPU(long, kasan_page_alloc_skip);
 
+/*
+ * Flush dcache after writing the tag for certain H/W to maintain cache coherence.
+ * The default value is chosen not to flush the cache.
+ */
+DEFINE_STATIC_KEY_FALSE(kasan_inval_dcache);
+
 /* kasan=off/on */
 static int __init early_kasan_flag(char *arg)
 {
@@ -191,6 +197,14 @@ static int __init early_kasan_flag_page_alloc_sample_order(char *arg)
 }
 early_param("kasan.page_alloc.sample.order", early_kasan_flag_page_alloc_sample_order);
 
+static int __init kasan_set_inval_dcache(char *arg)
+{
+	static_branch_enable(&kasan_inval_dcache);
+
+	return 0;
+}
+early_param("kasan_inval_dcache", kasan_set_inval_dcache);
+
 /*
  * kasan_init_hw_tags_cpu() is called for each CPU.
  * Not marked as __init as a CPU can be hot-plugged after boot.

mm/kasan/kasan.h

@@ -7,6 +7,7 @@
 #include <linux/kasan-tags.h>
 #include <linux/kfence.h>
 #include <linux/stackdepot.h>
+#include <asm/cacheflush.h>
 
 #if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)
 
@@ -468,25 +469,37 @@ static inline u8 kasan_random_tag(void) { return 0; }
 
 #ifdef CONFIG_KASAN_HW_TAGS
 
+DECLARE_STATIC_KEY_FALSE(kasan_inval_dcache);
+
 static inline void kasan_poison(const void *addr, size_t size, u8 value, bool init)
 {
+	addr = kasan_reset_tag(addr);
+
 	if (WARN_ON((unsigned long)addr & KASAN_GRANULE_MASK))
 		return;
 	if (WARN_ON(size & KASAN_GRANULE_MASK))
 		return;
 
-	hw_set_mem_tag_range(kasan_reset_tag(addr), size, value, init);
+	hw_set_mem_tag_range((void *)addr, size, value, init);
+
+	if (static_branch_unlikely(&kasan_inval_dcache) && size)
+		dcache_clean_inval_poc((unsigned long)addr, (unsigned long)addr + size);
 }
 
 static inline void kasan_unpoison(const void *addr, size_t size, bool init)
 {
 	u8 tag = get_tag(addr);
 
+	addr = kasan_reset_tag(addr);
+
 	if (WARN_ON((unsigned long)addr & KASAN_GRANULE_MASK))
 		return;
 	size = round_up(size, KASAN_GRANULE_SIZE);
 
-	hw_set_mem_tag_range(kasan_reset_tag(addr), size, tag, init);
+	hw_set_mem_tag_range((void *)addr, size, tag, init);
+
+	if (static_branch_unlikely(&kasan_inval_dcache) && size)
+		dcache_clean_inval_poc((unsigned long)addr, (unsigned long)addr + size);
 }
 
 static inline bool kasan_byte_accessible(const void *addr)