From a1dfc257ae631464fada5881052af7bce8b5533f Mon Sep 17 00:00:00 2001 From: Chris Goldsworthy Date: Tue, 5 Oct 2021 23:02:29 -0700 Subject: [PATCH] ANDROID: arm64/mm: Add command line option to make ZONE_DMA32 empty ZONE_DMA32 is enabled by default, yet it is not needed for all devices, nor is it desirable to have if not needed. For instance, if a partner in GKI 1.0 did not use ZONE_DMA32, memory can be lower for ZONE_NORMAL relative to older targets, such that memory would run out more quickly in ZONE_NORMAL leading kswapd to be invoked unnecessarily. Correspondingly, provide a means of making ZONE_DMA32 empty via the kernel command line when it is compiled in via CONFIG_ZONE_DMA32. P.S. The following two patches are squashed into this one, 1. bf96382 ("ANDROID: dma-direct: Make DMA32 disablement work for CONFIG_NUMA") 2. 135406c ("ANDROID: dma-direct: Document disable_dma32") Additionally, for kernels 6.6+, ensure that arm64_dma_phys_limit is set correctly by letting it be set to the highest possible physical address when disable_dma32 is present on the kernel command line. This allows SWIOTLB allocations to be satisfied from anywhere within the physical memory space, as opposed to restricting these allocations to the lower 32-bit addressable physical memory space. It is important that SWIOTLB allocations use the correct addressing limits, as SWIOTLB buffers can now be allocated at runtime--not just boot. Using the correct addressing limits can help increase the amount of memory available for SWIOTLB allocations, increasing the likelihood of success. This also aligns the behavior of disable_dma32 to what would happen if CONFIG_ZONE_DMA32 were disabled. Bug: 199917449 Bug: 268587627 Bug: 296282294 Bug: 372371001 Change-Id: I70ec76914b92e518d61a61072f0b3cb41cb28646 Signed-off-by: Chris Goldsworthy Signed-off-by: Sudarshan Rajagopalan Signed-off-by: Chinwen Chang [isaacmanjarres: Updated commit message to note that this version of the patch treats arm64_phys_dma_limit differently than the original version of the patch.] Signed-off-by: Isaac J. Manjarres Signed-off-by: Chinwen Chang --- .../admin-guide/kernel-parameters.txt | 4 +++ arch/arm64/mm/init.c | 26 +++++++++++++++--- arch/x86/mm/init.c | 20 +++++++++++++- include/linux/dma-direct.h | 27 +++++++++++++++++++ kernel/dma/direct.c | 6 +++-- kernel/dma/pool.c | 8 +++--- 6 files changed, 81 insertions(+), 10 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index fc20c834c9fe..f2651373f44a 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -1133,6 +1133,10 @@ can be useful when debugging issues that require an SLB miss to occur. + disable_dma32= [KNL] + Dynamically disable ZONE_DMA32 on kernels compiled with + CONFIG_ZONE_DMA32=y. + disable= [IPV6] See Documentation/networking/ipv6.rst. diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 27a32ff15412..8034121f0bd1 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -66,6 +66,12 @@ EXPORT_SYMBOL(memstart_addr); */ phys_addr_t __ro_after_init arm64_dma_phys_limit; +/* + * Provide a run-time mean of disabling ZONE_DMA32 if it is enabled via + * CONFIG_ZONE_DMA32. + */ +static bool disable_dma32 __ro_after_init; + /* * To make optimal use of block mappings when laying out the linear * mapping, round down the base of physical memory to a size that can @@ -144,9 +150,11 @@ static void __init zone_sizes_init(void) max_zone_pfns[ZONE_DMA] = PFN_DOWN(arm64_dma_phys_limit); #endif #ifdef CONFIG_ZONE_DMA32 - max_zone_pfns[ZONE_DMA32] = PFN_DOWN(dma32_phys_limit); - if (!arm64_dma_phys_limit) - arm64_dma_phys_limit = dma32_phys_limit; + if (!disable_dma32) { + max_zone_pfns[ZONE_DMA32] = PFN_DOWN(dma32_phys_limit); + if (!arm64_dma_phys_limit) + arm64_dma_phys_limit = dma32_phys_limit; + } #endif if (!arm64_dma_phys_limit) arm64_dma_phys_limit = PHYS_MASK + 1; @@ -155,6 +163,18 @@ static void __init zone_sizes_init(void) free_area_init(max_zone_pfns); } +static int __init early_disable_dma32(char *buf) +{ + if (!buf) + return -EINVAL; + + if (!strcmp(buf, "on")) + disable_dma32 = true; + + return 0; +} +early_param("disable_dma32", early_disable_dma32); + int pfn_is_map_memory(unsigned long pfn) { phys_addr_t addr = PFN_PHYS(pfn); diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c index eb503f53c319..26b7ebc0ac44 100644 --- a/arch/x86/mm/init.c +++ b/arch/x86/mm/init.c @@ -113,6 +113,12 @@ static unsigned long min_pfn_mapped; static bool __initdata can_use_brk_pgt = true; +/* + * Provide a run-time mean of disabling ZONE_DMA32 if it is enabled via + * CONFIG_ZONE_DMA32. + */ +static bool disable_dma32 __ro_after_init; + /* * Pages returned are already directly mapped. * @@ -997,7 +1003,7 @@ void __init zone_sizes_init(void) max_zone_pfns[ZONE_DMA] = min(MAX_DMA_PFN, max_low_pfn); #endif #ifdef CONFIG_ZONE_DMA32 - max_zone_pfns[ZONE_DMA32] = min(MAX_DMA32_PFN, max_low_pfn); + max_zone_pfns[ZONE_DMA32] = disable_dma32 ? 0 : min(MAX_DMA32_PFN, max_low_pfn); #endif max_zone_pfns[ZONE_NORMAL] = max_low_pfn; #ifdef CONFIG_HIGHMEM @@ -1007,6 +1013,18 @@ void __init zone_sizes_init(void) free_area_init(max_zone_pfns); } +static int __init early_disable_dma32(char *buf) +{ + if (!buf) + return -EINVAL; + + if (!strcmp(buf, "on")) + disable_dma32 = true; + + return 0; +} +early_param("disable_dma32", early_disable_dma32); + __visible DEFINE_PER_CPU_ALIGNED(struct tlb_state, cpu_tlbstate) = { .loaded_mm = &init_mm, .next_asid = 1, diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h index d7e30d4f7503..34e93632e1e9 100644 --- a/include/linux/dma-direct.h +++ b/include/linux/dma-direct.h @@ -23,6 +23,33 @@ struct bus_dma_region { u64 size; }; +static inline bool zone_dma32_is_empty(int node) +{ +#ifdef CONFIG_ZONE_DMA32 + pg_data_t *pgdat = NODE_DATA(node); + + return zone_is_empty(&pgdat->node_zones[ZONE_DMA32]); +#else + return true; +#endif +} + +static inline bool zone_dma32_are_empty(void) +{ +#ifdef CONFIG_NUMA + int node; + + for_each_node(node) + if (!zone_dma32_is_empty(node)) + return false; +#else + if (!zone_dma32_is_empty(numa_node_id())) + return false; +#endif + + return true; +} + static inline dma_addr_t translate_phys_to_dma(struct device *dev, phys_addr_t paddr) { diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index 5b4e6d3bf7bc..8db963c0d0cb 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -61,7 +61,8 @@ static gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 *phys_limit) *phys_limit = dma_to_phys(dev, dma_limit); if (*phys_limit <= zone_dma_limit) return GFP_DMA; - if (*phys_limit <= DMA_BIT_MASK(32)) + if (*phys_limit <= DMA_BIT_MASK(32) && + !zone_dma32_are_empty()) return GFP_DMA32; return 0; } @@ -145,7 +146,8 @@ again: if (IS_ENABLED(CONFIG_ZONE_DMA32) && phys_limit < DMA_BIT_MASK(64) && - !(gfp & (GFP_DMA32 | GFP_DMA))) { + !(gfp & (GFP_DMA32 | GFP_DMA)) && + !zone_dma32_are_empty()) { gfp |= GFP_DMA32; goto again; } diff --git a/kernel/dma/pool.c b/kernel/dma/pool.c index 7b04f7575796..0bd89a7cfa24 100644 --- a/kernel/dma/pool.c +++ b/kernel/dma/pool.c @@ -71,7 +71,7 @@ static bool cma_in_zone(gfp_t gfp) end = cma_get_base(cma) + size - 1; if (IS_ENABLED(CONFIG_ZONE_DMA) && (gfp & GFP_DMA)) return end <= zone_dma_limit; - if (IS_ENABLED(CONFIG_ZONE_DMA32) && (gfp & GFP_DMA32)) + if (IS_ENABLED(CONFIG_ZONE_DMA32) && (gfp & GFP_DMA32) && !zone_dma32_are_empty()) return end <= max(DMA_BIT_MASK(32), zone_dma_limit); return true; } @@ -153,7 +153,7 @@ static void atomic_pool_work_fn(struct work_struct *work) if (IS_ENABLED(CONFIG_ZONE_DMA)) atomic_pool_resize(atomic_pool_dma, GFP_KERNEL | GFP_DMA); - if (IS_ENABLED(CONFIG_ZONE_DMA32)) + if (IS_ENABLED(CONFIG_ZONE_DMA32) && !zone_dma32_are_empty()) atomic_pool_resize(atomic_pool_dma32, GFP_KERNEL | GFP_DMA32); atomic_pool_resize(atomic_pool_kernel, GFP_KERNEL); @@ -209,7 +209,7 @@ static int __init dma_atomic_pool_init(void) if (!atomic_pool_dma) ret = -ENOMEM; } - if (IS_ENABLED(CONFIG_ZONE_DMA32)) { + if (IS_ENABLED(CONFIG_ZONE_DMA32) && !zone_dma32_are_empty()) { atomic_pool_dma32 = __dma_atomic_pool_init(atomic_pool_size, GFP_KERNEL | GFP_DMA32); if (!atomic_pool_dma32) @@ -224,7 +224,7 @@ postcore_initcall(dma_atomic_pool_init); static inline struct gen_pool *dma_guess_pool(struct gen_pool *prev, gfp_t gfp) { if (prev == NULL) { - if (IS_ENABLED(CONFIG_ZONE_DMA32) && (gfp & GFP_DMA32)) + if (IS_ENABLED(CONFIG_ZONE_DMA32) && (gfp & GFP_DMA32) && !zone_dma32_are_empty()) return atomic_pool_dma32; if (atomic_pool_dma && (gfp & GFP_DMA)) return atomic_pool_dma;