diff --git a/Documentation/arch/arm/kernel_mode_neon.rst b/Documentation/arch/arm/kernel_mode_neon.rst index 9bfb71a2a9b9..1efb6d35b7bd 100644 --- a/Documentation/arch/arm/kernel_mode_neon.rst +++ b/Documentation/arch/arm/kernel_mode_neon.rst @@ -121,4 +121,6 @@ observe the following in addition to the rules above: * Compile the unit containing the NEON intrinsics with '-ffreestanding' so GCC uses its builtin version of (this is a C99 header which the kernel does not supply); -* Include last, or at least after +* Do not include directly: instead, include , + which tweaks some macro definitions so that system headers can be included + safely. diff --git a/MAINTAINERS b/MAINTAINERS index 226f4a570394..2001a1b93176 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -13952,7 +13952,7 @@ F: scripts/Makefile.randstruct F: security/Kconfig.hardening K: \b(add|choose)_random_kstack_offset\b K: \b__check_(object_size|heap_object)\b -K: \b__counted_by(_le|_be)?\b +K: \b__counted_by(_le|_be|_ptr)?\b KERNEL JANITORS L: kernel-janitors@vger.kernel.org diff --git a/arch/arm/include/asm/neon-intrinsics.h b/arch/arm/include/asm/neon-intrinsics.h new file mode 100644 index 000000000000..8b80c05ce1d7 --- /dev/null +++ b/arch/arm/include/asm/neon-intrinsics.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __ASM_NEON_INTRINSICS_H +#define __ASM_NEON_INTRINSICS_H + +#ifndef __ARM_NEON__ +#error You should compile this file with '-march=armv7-a -mfloat-abi=softfp -mfpu=neon' +#endif + +#include + +/* + * The C99 types uintXX_t that are usually defined in 'stdint.h' are not as + * unambiguous on ARM as you would expect. For the types below, there is a + * difference on ARM between GCC built for bare metal ARM, GCC built for glibc + * and the kernel itself, which results in build errors if you try to build + * with -ffreestanding and include 'stdint.h' (such as when you include + * 'arm_neon.h' in order to use NEON intrinsics) + * + * As the typedefs for these types in 'stdint.h' are based on builtin defines + * supplied by GCC, we can tweak these to align with the kernel's idea of those + * types, so 'linux/types.h' and 'stdint.h' can be safely included from the + * same source file (provided that -ffreestanding is used). + * + * int32_t uint32_t intptr_t uintptr_t + * bare metal GCC long unsigned long int unsigned int + * glibc GCC int unsigned int int unsigned int + * kernel int unsigned int long unsigned long + */ + +#ifdef __INT32_TYPE__ +#undef __INT32_TYPE__ +#define __INT32_TYPE__ int +#endif + +#ifdef __UINT32_TYPE__ +#undef __UINT32_TYPE__ +#define __UINT32_TYPE__ unsigned int +#endif + +#ifdef __INTPTR_TYPE__ +#undef __INTPTR_TYPE__ +#define __INTPTR_TYPE__ long +#endif + +#ifdef __UINTPTR_TYPE__ +#undef __UINTPTR_TYPE__ +#define __UINTPTR_TYPE__ unsigned long +#endif + +/* + * genksyms chokes on the ARM NEON instrinsics system header, but we + * don't export anything it defines anyway, so just disregard when + * genksyms execute. + */ +#ifndef __GENKSYMS__ +#include +#endif + +#endif /* __ASM_NEON_INTRINSICS_H */ diff --git a/crypto/Makefile b/crypto/Makefile index f98f57c7a49f..0798fd7aea0d 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -103,13 +103,14 @@ obj-$(CONFIG_CRYPTO_CHACHA20POLY1305) += chacha20poly1305.o obj-$(CONFIG_CRYPTO_AEGIS128) += aegis128.o aegis128-y := aegis128-core.o +CFLAGS_aegis128-neon-inner.o += $(CC_FLAGS_FPU) +CFLAGS_REMOVE_aegis128-neon-inner.o += $(CC_FLAGS_NO_FPU) ifeq ($(ARCH),arm) -CFLAGS_aegis128-neon-inner.o += -ffreestanding -march=armv8-a -mfloat-abi=softfp -CFLAGS_aegis128-neon-inner.o += -mfpu=crypto-neon-fp-armv8 +CFLAGS_aegis128-neon-inner.o += -march=armv8-a -mfpu=crypto-neon-fp-armv8 aegis128-$(CONFIG_CRYPTO_AEGIS128_SIMD) += aegis128-neon.o aegis128-neon-inner.o endif ifeq ($(ARCH),arm64) -aegis128-cflags-y := -ffreestanding -mcpu=generic+crypto +aegis128-cflags-y := -mcpu=generic+crypto aegis128-cflags-$(CONFIG_CC_IS_GCC) += -ffixed-q16 -ffixed-q17 -ffixed-q18 \ -ffixed-q19 -ffixed-q20 -ffixed-q21 \ -ffixed-q22 -ffixed-q23 -ffixed-q24 \ @@ -117,11 +118,8 @@ aegis128-cflags-$(CONFIG_CC_IS_GCC) += -ffixed-q16 -ffixed-q17 -ffixed-q18 \ -ffixed-q28 -ffixed-q29 -ffixed-q30 \ -ffixed-q31 CFLAGS_aegis128-neon-inner.o += $(aegis128-cflags-y) -CFLAGS_REMOVE_aegis128-neon-inner.o += -mgeneral-regs-only aegis128-$(CONFIG_CRYPTO_AEGIS128_SIMD) += aegis128-neon.o aegis128-neon-inner.o endif -# Enable -CFLAGS_aegis128-neon-inner.o += -isystem $(shell $(CC) -print-file-name=include) obj-$(CONFIG_CRYPTO_PCRYPT) += pcrypt.o obj-$(CONFIG_CRYPTO_CRYPTD) += cryptd.o diff --git a/crypto/aegis128-neon-inner.c b/crypto/aegis128-neon-inner.c index b6a52a386b22..56b534eeb680 100644 --- a/crypto/aegis128-neon-inner.c +++ b/crypto/aegis128-neon-inner.c @@ -3,13 +3,11 @@ * Copyright (C) 2019 Linaro, Ltd. */ -#ifdef CONFIG_ARM64 #include +#ifdef CONFIG_ARM64 #define AES_ROUND "aese %0.16b, %1.16b \n\t aesmc %0.16b, %0.16b" #else -#include - #define AES_ROUND "aese.8 %q0, %q1 \n\t aesmc.8 %q0, %q0" #endif diff --git a/drivers/misc/lkdtm/Makefile b/drivers/misc/lkdtm/Makefile index 03ebe33185f9..4e58d16fc01e 100644 --- a/drivers/misc/lkdtm/Makefile +++ b/drivers/misc/lkdtm/Makefile @@ -11,7 +11,7 @@ lkdtm-$(CONFIG_LKDTM) += usercopy.o lkdtm-$(CONFIG_LKDTM) += kstack_erase.o lkdtm-$(CONFIG_LKDTM) += cfi.o lkdtm-$(CONFIG_LKDTM) += fortify.o -lkdtm-$(CONFIG_PPC_64S_HASH_MMU) += powerpc.o +lkdtm-$(CONFIG_PPC_BOOK3S_64) += powerpc.o KASAN_SANITIZE_stackleak.o := n diff --git a/drivers/misc/lkdtm/bugs.c b/drivers/misc/lkdtm/bugs.c index e0098f314570..3eca2ef64aff 100644 --- a/drivers/misc/lkdtm/bugs.c +++ b/drivers/misc/lkdtm/bugs.c @@ -7,6 +7,7 @@ */ #include "lkdtm.h" #include +#include #include #include #include @@ -817,6 +818,29 @@ static noinline void lkdtm_CORRUPT_PAC(void) #endif } +static void __maybe_unused lkdtm_EFI_RUNTIME_CRASH(void) +{ + static unsigned long size __ro_after_init = sizeof(efi_char16_t); + efi_status_t status; + + if (!efi.get_next_variable || + !efi_enabled(EFI_RUNTIME_SERVICES) || + !efi_rt_services_supported(EFI_RT_SUPPORTED_GET_NEXT_VARIABLE_NAME)) { + pr_err("FAIL: EFI GetNextVariableName() is not available\n"); + return; + } + + /* + * Provoke a fault by asking the firmware to write to a read-only + * variable. + */ + status = efi.get_next_variable(&size, L"", &(efi_guid_t){}); + + if (status != EFI_ABORTED || efi_enabled(EFI_RUNTIME_SERVICES)) + pr_err("FAIL: EFI GetNextVariable() did not abort (%#lx)\n", + status); +} + static struct crashtype crashtypes[] = { CRASHTYPE(PANIC), CRASHTYPE(PANIC_STOP_IRQOFF), @@ -850,6 +874,9 @@ static struct crashtype crashtypes[] = { CRASHTYPE(UNSET_SMEP), CRASHTYPE(DOUBLE_FAULT), CRASHTYPE(CORRUPT_PAC), +#ifdef CONFIG_EFI + CRASHTYPE(EFI_RUNTIME_CRASH), +#endif }; struct crashtype_category bugs_crashtypes = { diff --git a/drivers/misc/lkdtm/core.c b/drivers/misc/lkdtm/core.c index 5732fd59a227..ededa32d6744 100644 --- a/drivers/misc/lkdtm/core.c +++ b/drivers/misc/lkdtm/core.c @@ -96,7 +96,7 @@ static const struct crashtype_category *crashtype_categories[] = { &stackleak_crashtypes, &cfi_crashtypes, &fortify_crashtypes, -#ifdef CONFIG_PPC_64S_HASH_MMU +#ifdef CONFIG_PPC_BOOK3S_64 &powerpc_crashtypes, #endif }; diff --git a/drivers/misc/lkdtm/powerpc.c b/drivers/misc/lkdtm/powerpc.c index be385449911a..6eaac79ea26b 100644 --- a/drivers/misc/lkdtm/powerpc.c +++ b/drivers/misc/lkdtm/powerpc.c @@ -5,6 +5,7 @@ #include #include +#ifdef CONFIG_PPC_64S_HASH_MMU /* Inserts new slb entries */ static void insert_slb_entry(unsigned long p, int ssize, int page_size) { @@ -17,11 +18,14 @@ static void insert_slb_entry(unsigned long p, int ssize, int page_size) : "r" (mk_vsid_data(p, ssize, flags)), "r" (mk_esid_data(p, ssize, SLB_NUM_BOLTED)) : "memory"); + isync(); asm volatile("slbmte %0,%1" : : "r" (mk_vsid_data(p, ssize, flags)), "r" (mk_esid_data(p, ssize, SLB_NUM_BOLTED + 1)) : "memory"); + isync(); + preempt_enable(); } @@ -84,6 +88,7 @@ static void insert_dup_slb_entry_0(void) : "r" (vsid), "r" (esid | SLB_NUM_BOLTED) : "memory"); + isync(); asm volatile("slbmfee %0,%1" : "=r" (esid) : "r" (i)); asm volatile("slbmfev %0,%1" : "=r" (vsid) : "r" (i)); @@ -93,15 +98,42 @@ static void insert_dup_slb_entry_0(void) : "r" (vsid), "r" (esid | (SLB_NUM_BOLTED + 1)) : "memory"); + isync(); pr_info("%s accessing test address 0x%lx: 0x%lx\n", __func__, test_address, *test_ptr); preempt_enable(); } +#endif /* CONFIG_PPC_64S_HASH_MMU */ + +static __always_inline void tlbiel_va(unsigned long va, + unsigned long pid, + unsigned long ap, + unsigned long ric) +{ + unsigned long rb, rs, prs, r; + + rb = va & ~(PPC_BITMASK(52, 63)); + rb |= ap << PPC_BITLSHIFT(58); + rs = pid << PPC_BITLSHIFT(31); + + prs = 1; /* process scoped */ + r = 1; /* radix format */ + + /* + * Trigger an MCE by issuing radix tlbiel with an invalid operand combination. + * The combination of RIC = 2 with IS = 0 (Invalidation selector specified + * in the RB register) is invalid. + * This invalid combination causes hardware to raise a machine check. + */ + asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1) + : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); +} static void lkdtm_PPC_SLB_MULTIHIT(void) { +#ifdef CONFIG_PPC_64S_HASH_MMU if (!radix_enabled()) { pr_info("Injecting SLB multihit errors\n"); /* @@ -117,10 +149,27 @@ static void lkdtm_PPC_SLB_MULTIHIT(void) } else { pr_err("XFAIL: This test is for ppc64 and with hash mode MMU only\n"); } +#else + pr_err("XFAIL: This test requires CONFIG_PPC_64S_HASH_MMU\n"); +#endif +} + +static void lkdtm_PPC_RADIX_TLBIEL(void) +{ + unsigned long addr = PAGE_OFFSET; + + if (radix_enabled()) { + pr_info("Injecting Radix TLB invalidation MCE\n"); + tlbiel_va(addr, 0, 0, RIC_FLUSH_ALL); + pr_info("Recovered from radix tlbiel attempt\n"); + } else { + pr_err("XFAIL: This test is for ppc64 and with radix mode MMU only\n"); + } } static struct crashtype crashtypes[] = { CRASHTYPE(PPC_SLB_MULTIHIT), + CRASHTYPE(PPC_RADIX_TLBIEL), }; struct crashtype_category powerpc_crashtypes = { diff --git a/fs/btrfs/Kconfig b/fs/btrfs/Kconfig index 5e75438e0b73..9de04c37e11a 100644 --- a/fs/btrfs/Kconfig +++ b/fs/btrfs/Kconfig @@ -93,10 +93,6 @@ config BTRFS_EXPERIMENTAL Current list: - - COW fixup worker warning - last warning before removing the - functionality catching out-of-band page - dirtying, not necessary since 5.8 - - RAID mirror read policy - additional read policies for balancing reading from redundant block group profiles (currently: pid, round-robin, @@ -110,7 +106,9 @@ config BTRFS_EXPERIMENTAL - extent tree v2 - complex rework of extent tracking - - large folio and block size (> page size) support + - block size > page size support + + - huge folios for data - folios can be as large as 2MiB now - asynchronous checksum generation for data writes diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c index 273924ca912c..23c3eeb58dc1 100644 --- a/fs/btrfs/backref.c +++ b/fs/btrfs/backref.c @@ -1256,7 +1256,7 @@ static bool lookup_backref_shared_cache(struct btrfs_backref_share_check_ctx *ct * realizing. We cache results only for extent buffers that lead from * the root node down to the leaf with the file extent item. */ - ASSERT(level >= 0); + ASSERT(level >= 0, "level=%d", level); entry = &ctx->path_cache_entries[level]; @@ -1327,7 +1327,7 @@ static void store_backref_shared_cache(struct btrfs_backref_share_check_ctx *ctx * realizing. We cache results only for extent buffers that lead from * the root node down to the leaf with the file extent item. */ - ASSERT(level >= 0); + ASSERT(level >= 0, "level=%d", level); if (is_shared) gen = btrfs_get_last_root_drop_gen(fs_info); @@ -2367,7 +2367,7 @@ int tree_backref_for_extent(unsigned long *ptr, struct extent_buffer *eb, info = (struct btrfs_tree_block_info *)(ei + 1); *out_level = btrfs_tree_block_level(eb, info); } else { - ASSERT(key->type == BTRFS_METADATA_ITEM_KEY); + ASSERT(key->type == BTRFS_METADATA_ITEM_KEY, "key->type=%hhu", key->type); *out_level = (u8)key->offset; } @@ -2814,26 +2814,17 @@ struct inode_fs_paths *init_ipath(s32 total_bytes, struct btrfs_root *fs_root, return ifp; } -struct btrfs_backref_iter *btrfs_backref_iter_alloc(struct btrfs_fs_info *fs_info) +int btrfs_backref_iter_init(struct btrfs_backref_iter *iter) { - struct btrfs_backref_iter *ret; - - ret = kzalloc_obj(*ret, GFP_NOFS); - if (!ret) - return NULL; - - ret->path = btrfs_alloc_path(); - if (!ret->path) { - kfree(ret); - return NULL; - } + iter->path = btrfs_alloc_path(); + if (!iter->path) + return -ENOMEM; /* Current backref iterator only supports iteration in commit root */ - ret->path->search_commit_root = true; - ret->path->skip_locking = true; - ret->fs_info = fs_info; + iter->path->search_commit_root = true; + iter->path->skip_locking = true; - return ret; + return 0; } static void btrfs_backref_iter_release(struct btrfs_backref_iter *iter) @@ -2846,9 +2837,8 @@ static void btrfs_backref_iter_release(struct btrfs_backref_iter *iter) memset(&iter->cur_key, 0, sizeof(iter->cur_key)); } -int btrfs_backref_iter_start(struct btrfs_backref_iter *iter, u64 bytenr) +int btrfs_backref_iter_start(struct btrfs_fs_info *fs_info, struct btrfs_backref_iter *iter, u64 bytenr) { - struct btrfs_fs_info *fs_info = iter->fs_info; struct btrfs_root *extent_root = btrfs_extent_root(fs_info, bytenr); struct btrfs_path *path = iter->path; struct btrfs_extent_item *ei; @@ -2963,7 +2953,7 @@ static bool btrfs_backref_iter_is_inline_ref(struct btrfs_backref_iter *iter) * Return >0 if there is no extra backref for this bytenr. * Return <0 if there is something wrong happened. */ -int btrfs_backref_iter_next(struct btrfs_backref_iter *iter) +int btrfs_backref_iter_next(struct btrfs_fs_info *fs_info, struct btrfs_backref_iter *iter) { struct extent_buffer *eb = iter->path->nodes[0]; struct btrfs_root *extent_root; @@ -2974,7 +2964,9 @@ int btrfs_backref_iter_next(struct btrfs_backref_iter *iter) if (btrfs_backref_iter_is_inline_ref(iter)) { /* We're still inside the inline refs */ - ASSERT(iter->cur_ptr < iter->end_ptr); + ASSERT(iter->cur_ptr < iter->end_ptr, + "iter->cur_ptr=%u iter->end_ptr=%u", + iter->cur_ptr, iter->end_ptr); if (btrfs_backref_has_tree_block_info(iter)) { /* First tree block info */ @@ -2997,10 +2989,9 @@ int btrfs_backref_iter_next(struct btrfs_backref_iter *iter) } /* We're at keyed items, there is no inline item, go to the next one */ - extent_root = btrfs_extent_root(iter->fs_info, iter->bytenr); + extent_root = btrfs_extent_root(fs_info, iter->bytenr); if (unlikely(!extent_root)) { - btrfs_err(iter->fs_info, - "missing extent root for extent at bytenr %llu", + btrfs_err(fs_info, "missing extent root for extent at bytenr %llu", iter->bytenr); return -EUCLEAN; } @@ -3041,7 +3032,7 @@ struct btrfs_backref_node *btrfs_backref_alloc_node( { struct btrfs_backref_node *node; - ASSERT(level >= 0 && level < BTRFS_MAX_LEVEL); + ASSERT(level >= 0 && level < BTRFS_MAX_LEVEL, "level=%d", level); node = kzalloc_obj(*node, GFP_NOFS); if (!node) return node; @@ -3063,7 +3054,7 @@ void btrfs_backref_free_node(struct btrfs_backref_cache *cache, if (node) { ASSERT(list_empty(&node->list)); ASSERT(list_empty(&node->lower)); - ASSERT(node->eb == NULL); + ASSERT(node->eb == NULL, "node->eb->start=%llu", node->eb->start); cache->nr_nodes--; btrfs_put_root(node->root); kfree(node); @@ -3166,15 +3157,18 @@ void btrfs_backref_release_cache(struct btrfs_backref_cache *cache) ASSERT(list_empty(&cache->pending_edge)); ASSERT(list_empty(&cache->useless_node)); - ASSERT(!cache->nr_nodes); - ASSERT(!cache->nr_edges); + ASSERT(!cache->nr_nodes, "cache->nr_nodes=%d", cache->nr_nodes); + ASSERT(!cache->nr_edges, "cache->nr_edges=%d", cache->nr_edges); } static void btrfs_backref_link_edge(struct btrfs_backref_edge *edge, struct btrfs_backref_node *lower, struct btrfs_backref_node *upper) { - ASSERT(upper && lower && upper->level == lower->level + 1); + ASSERT(upper != NULL); + ASSERT(lower != NULL); + ASSERT(upper->level == lower->level + 1, "upper->level=%d lower->level=%d", + upper->level, lower->level); edge->node[LOWER] = lower; edge->node[UPPER] = upper; list_add_tail(&edge->list[LOWER], &lower->upper); @@ -3199,7 +3193,7 @@ static int handle_direct_tree_backref(struct btrfs_backref_cache *cache, struct btrfs_backref_node *upper; struct rb_node *rb_node; - ASSERT(ref_key->type == BTRFS_SHARED_BLOCK_REF_KEY); + ASSERT(ref_key->type == BTRFS_SHARED_BLOCK_REF_KEY, "ref_key->type=%hhu", ref_key->type); /* Only reloc root uses backref pointing to itself */ if (ref_key->objectid == ref_key->offset) { @@ -3294,7 +3288,9 @@ static int handle_indirect_tree_backref(struct btrfs_trans_handle *trans, if (btrfs_root_level(&root->root_item) == cur->level) { /* Tree root */ - ASSERT(btrfs_root_bytenr(&root->root_item) == cur->bytenr); + ASSERT(btrfs_root_bytenr(&root->root_item) == cur->bytenr, + "root_bytenr=%llu cur->bytenr=%llu", + btrfs_root_bytenr(&root->root_item), cur->bytenr); /* * For reloc backref cache, we may ignore reloc root. But for * general purpose backref cache, we can't rely on @@ -3344,8 +3340,9 @@ static int handle_indirect_tree_backref(struct btrfs_trans_handle *trans, /* Add all nodes and edges in the path */ for (; level < BTRFS_MAX_LEVEL; level++) { if (!path->nodes[level]) { - ASSERT(btrfs_root_bytenr(&root->root_item) == - lower->bytenr); + ASSERT(btrfs_root_bytenr(&root->root_item) == lower->bytenr, + "root_bytenr=%llu lower->bytenr=%llu", + btrfs_root_bytenr(&root->root_item), lower->bytenr); /* Same as previous should_ignore_reloc_root() call */ if (btrfs_should_ignore_reloc_root(root) && cache->is_reloc) { @@ -3454,7 +3451,7 @@ int btrfs_backref_add_tree_node(struct btrfs_trans_handle *trans, struct btrfs_backref_node *exist; int ret; - ret = btrfs_backref_iter_start(iter, cur->bytenr); + ret = btrfs_backref_iter_start(trans->fs_info, iter, cur->bytenr); if (ret < 0) return ret; /* @@ -3462,7 +3459,7 @@ int btrfs_backref_add_tree_node(struct btrfs_trans_handle *trans, * stored in it, but fetch it from the tree block */ if (btrfs_backref_has_tree_block_info(iter)) { - ret = btrfs_backref_iter_next(iter); + ret = btrfs_backref_iter_next(trans->fs_info, iter); if (ret < 0) goto out; /* No extra backref? This means the tree block is corrupted */ @@ -3492,7 +3489,7 @@ int btrfs_backref_add_tree_node(struct btrfs_trans_handle *trans, exist = NULL; } - for (; ret == 0; ret = btrfs_backref_iter_next(iter)) { + for (; ret == 0; ret = btrfs_backref_iter_next(trans->fs_info, iter)) { struct extent_buffer *eb; struct btrfs_key key; int type; diff --git a/fs/btrfs/backref.h b/fs/btrfs/backref.h index 1d009b0f4c69..179791de6b19 100644 --- a/fs/btrfs/backref.h +++ b/fs/btrfs/backref.h @@ -278,15 +278,12 @@ struct prelim_ref { struct btrfs_backref_iter { u64 bytenr; struct btrfs_path *path; - struct btrfs_fs_info *fs_info; struct btrfs_key cur_key; u32 item_ptr; u32 cur_ptr; u32 end_ptr; }; -struct btrfs_backref_iter *btrfs_backref_iter_alloc(struct btrfs_fs_info *fs_info); - /* * For metadata with EXTENT_ITEM key (non-skinny) case, the first inline data * is btrfs_tree_block_info, without a btrfs_extent_inline_ref header. @@ -302,9 +299,11 @@ static inline bool btrfs_backref_has_tree_block_info( return false; } -int btrfs_backref_iter_start(struct btrfs_backref_iter *iter, u64 bytenr); +int btrfs_backref_iter_init(struct btrfs_backref_iter *iter); -int btrfs_backref_iter_next(struct btrfs_backref_iter *iter); +int btrfs_backref_iter_start(struct btrfs_fs_info *fs_info, struct btrfs_backref_iter *iter, u64 bytenr); + +int btrfs_backref_iter_next(struct btrfs_fs_info *fs_info, struct btrfs_backref_iter *iter); /* * Backref cache related structures diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index b611c64119db..ab76a5173272 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -22,6 +22,34 @@ #include "accessors.h" #include "extent-tree.h" +static struct kmem_cache *block_group_cache; +static struct kmem_cache *free_space_ctl_cache; + +int __init btrfs_init_block_group(void) +{ + block_group_cache = kmem_cache_create("btrfs_block_group", + sizeof(struct btrfs_block_group), + 0, 0, NULL); + if (!block_group_cache) + return -ENOMEM; + + free_space_ctl_cache = kmem_cache_create("btrfs_free_space_ctl", + sizeof(struct btrfs_free_space_ctl), + 0, 0, NULL); + if (!free_space_ctl_cache) { + kmem_cache_destroy(block_group_cache); + return -ENOMEM; + } + + return 0; +} + +void __cold btrfs_exit_block_group(void) +{ + kmem_cache_destroy(block_group_cache); + kmem_cache_destroy(free_space_ctl_cache); +} + #ifdef CONFIG_BTRFS_DEBUG int btrfs_should_fragment_free_space(const struct btrfs_block_group *block_group) { @@ -180,9 +208,9 @@ void btrfs_put_block_group(struct btrfs_block_group *cache) btrfs_discard_cancel_work(&cache->fs_info->discard_ctl, cache); - kfree(cache->free_space_ctl); + kmem_cache_free(free_space_ctl_cache, cache->free_space_ctl); btrfs_free_chunk_map(cache->physical_map); - kfree(cache); + kmem_cache_free(block_group_cache, cache); } } @@ -2371,13 +2399,13 @@ static struct btrfs_block_group *btrfs_create_block_group( { struct btrfs_block_group *cache; - cache = kzalloc_obj(*cache, GFP_NOFS); + cache = kmem_cache_zalloc(block_group_cache, GFP_NOFS); if (!cache) return NULL; - cache->free_space_ctl = kzalloc_obj(*cache->free_space_ctl, GFP_NOFS); + cache->free_space_ctl = kmem_cache_zalloc(free_space_ctl_cache, GFP_NOFS); if (!cache->free_space_ctl) { - kfree(cache); + kmem_cache_free(block_group_cache, cache); return NULL; } @@ -2454,7 +2482,7 @@ static int check_chunk_block_group_mappings(struct btrfs_fs_info *fs_info) static int read_one_block_group(struct btrfs_fs_info *info, struct btrfs_block_group_item_v2 *bgi, const struct btrfs_key *key, - int need_clear) + bool need_clear) { struct btrfs_block_group *cache; const bool mixed = btrfs_fs_incompat(info, MIXED_GROUPS); @@ -2635,7 +2663,7 @@ int btrfs_read_block_groups(struct btrfs_fs_info *info) struct btrfs_block_group *cache; struct btrfs_space_info *space_info; struct btrfs_key key; - int need_clear = 0; + bool need_clear = false; u64 cache_gen; /* @@ -2660,9 +2688,9 @@ int btrfs_read_block_groups(struct btrfs_fs_info *info) cache_gen = btrfs_super_cache_generation(info->super_copy); if (btrfs_test_opt(info, SPACE_CACHE) && btrfs_super_generation(info->super_copy) != cache_gen) - need_clear = 1; + need_clear = true; if (btrfs_test_opt(info, CLEAR_CACHE)) - need_clear = 1; + need_clear = true; while (1) { struct btrfs_block_group_item_v2 bgi; @@ -4089,7 +4117,7 @@ int btrfs_force_chunk_alloc(struct btrfs_trans_handle *trans, u64 type) struct btrfs_space_info *space_info; space_info = btrfs_find_space_info(trans->fs_info, type); - if (!space_info) { + if (unlikely(!space_info)) { DEBUG_WARN(); return -EINVAL; } diff --git a/fs/btrfs/block-group.h b/fs/btrfs/block-group.h index 0504cb357992..790c2d467af5 100644 --- a/fs/btrfs/block-group.h +++ b/fs/btrfs/block-group.h @@ -122,6 +122,7 @@ struct btrfs_block_group { struct btrfs_fs_info *fs_info; struct btrfs_inode *inode; spinlock_t lock; + unsigned int ro; u64 start; u64 length; u64 pinned; @@ -134,7 +135,8 @@ struct btrfs_block_group { u64 global_root_id; u64 remap_bytes; u32 identity_remap_count; - + /* The last commited identity_remap_count value of this block group. */ + u32 last_identity_remap_count; /* * The last committed used bytes of this block group, if the above @used * is still the same as @last_used, we don't need to update block @@ -143,8 +145,6 @@ struct btrfs_block_group { u64 last_used; /* The last committed remap_bytes value of this block group. */ u64 last_remap_bytes; - /* The last commited identity_remap_count value of this block group. */ - u32 last_identity_remap_count; /* The last committed flags value for this block group. */ u64 last_flags; @@ -171,12 +171,10 @@ struct btrfs_block_group { unsigned long full_stripe_len; unsigned long runtime_flags; - unsigned int ro; - - int disk_cache_state; + enum btrfs_disk_cache_state disk_cache_state; /* Cache tracking stuff */ - int cached; + enum btrfs_caching_type cached; struct btrfs_caching_control *caching_ctl; struct btrfs_space_info *space_info; @@ -192,6 +190,16 @@ struct btrfs_block_group { refcount_t refs; + /* + * When non-zero it means the block group's logical address and its + * device extents can not be reused for future block group allocations + * until the counter goes down to 0. This is to prevent them from being + * reused while some task is still using the block group after it was + * deleted - we want to make sure they can only be reused for new block + * groups after that task is done with the deleted block group. + */ + atomic_t frozen; + /* * List of struct btrfs_free_clusters for this block group. * Today it will only have one thing on it, but that may change @@ -211,22 +219,12 @@ struct btrfs_block_group { /* For read-only block groups */ struct list_head ro_list; - /* - * When non-zero it means the block group's logical address and its - * device extents can not be reused for future block group allocations - * until the counter goes down to 0. This is to prevent them from being - * reused while some task is still using the block group after it was - * deleted - we want to make sure they can only be reused for new block - * groups after that task is done with the deleted block group. - */ - atomic_t frozen; - /* For discard operations */ struct list_head discard_list; int discard_index; + enum btrfs_discard_state discard_state; u64 discard_eligible_time; u64 discard_cursor; - enum btrfs_discard_state discard_state; /* For dirty block groups */ struct list_head dirty_list; @@ -263,6 +261,8 @@ struct btrfs_block_group { /* Protected by @free_space_lock. */ bool using_free_space_bitmaps_cached; + enum btrfs_block_group_size_class size_class:8; + /* * Number of extents in this block group used for swap files. * All accesses protected by the spinlock 'lock'. @@ -281,7 +281,6 @@ struct btrfs_block_group { struct list_head active_bg_list; struct work_struct zone_finish_work; struct extent_buffer *last_eb; - enum btrfs_block_group_size_class size_class; u64 reclaim_mark; }; @@ -320,6 +319,9 @@ static inline u64 btrfs_block_group_available_space(const struct btrfs_block_gro int btrfs_should_fragment_free_space(const struct btrfs_block_group *block_group); #endif +int __init btrfs_init_block_group(void); +void __cold btrfs_exit_block_group(void); + struct btrfs_block_group *btrfs_lookup_first_block_group( struct btrfs_fs_info *info, u64 bytenr); struct btrfs_block_group *btrfs_lookup_block_group( diff --git a/fs/btrfs/btrfs_inode.h b/fs/btrfs/btrfs_inode.h index 55c272fe5d92..d5d81f9546c3 100644 --- a/fs/btrfs/btrfs_inode.h +++ b/fs/btrfs/btrfs_inode.h @@ -128,15 +128,6 @@ struct btrfs_inode { /* which subvolume this inode belongs to */ struct btrfs_root *root; -#if BITS_PER_LONG == 32 - /* - * The objectid of the corresponding BTRFS_INODE_ITEM_KEY. - * On 64 bits platforms we can get it from vfs_inode.i_ino, which is an - * unsigned long and therefore 64 bits on such platforms. - */ - u64 objectid; -#endif - /* Cached value of inode property 'compression'. */ u8 prop_compress; @@ -372,30 +363,11 @@ static inline unsigned long btrfs_inode_hash(u64 objectid, return (unsigned long)h; } -#if BITS_PER_LONG == 32 - -/* - * On 32 bit systems the i_ino of struct inode is 32 bits (unsigned long), so - * we use the inode's location objectid which is a u64 to avoid truncation. - */ -static inline u64 btrfs_ino(const struct btrfs_inode *inode) -{ - u64 ino = inode->objectid; - - if (test_bit(BTRFS_INODE_ROOT_STUB, &inode->runtime_flags)) - ino = inode->vfs_inode.i_ino; - return ino; -} - -#else - static inline u64 btrfs_ino(const struct btrfs_inode *inode) { return inode->vfs_inode.i_ino; } -#endif - static inline void btrfs_get_inode_key(const struct btrfs_inode *inode, struct btrfs_key *key) { @@ -406,9 +378,6 @@ static inline void btrfs_get_inode_key(const struct btrfs_inode *inode, static inline void btrfs_set_inode_number(struct btrfs_inode *inode, u64 ino) { -#if BITS_PER_LONG == 32 - inode->objectid = ino; -#endif inode->vfs_inode.i_ino = ino; } @@ -531,12 +500,9 @@ static inline void btrfs_set_inode_mapping_order(struct btrfs_inode *inode) /* Metadata inode should not reach here. */ ASSERT(is_data_inode(inode)); - /* We only allow BITS_PER_LONGS blocks for each bitmap. */ -#ifdef CONFIG_BTRFS_EXPERIMENTAL mapping_set_folio_order_range(inode->vfs_inode.i_mapping, inode->root->fs_info->block_min_order, inode->root->fs_info->block_max_order); -#endif } void btrfs_calculate_block_csum_folio(struct btrfs_fs_info *fs_info, @@ -569,6 +535,8 @@ int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, long nr, int btrfs_set_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end, unsigned int extra_bits, struct extent_state **cached_state); +int btrfs_reset_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end, + unsigned int extra_bits, struct extent_state **cached_state); struct btrfs_new_inode_args { /* Input */ @@ -630,7 +598,6 @@ int btrfs_prealloc_file_range_trans(struct inode *inode, loff_t actual_len, u64 *alloc_hint); int btrfs_run_delalloc_range(struct btrfs_inode *inode, struct folio *locked_folio, u64 start, u64 end, struct writeback_control *wbc); -int btrfs_writepage_cow_fixup(struct folio *folio); int btrfs_encoded_io_compression_from_extent(struct btrfs_fs_info *fs_info, int compress_type); int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode, diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index a02b62e0a8f3..ffb6b52863a7 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -355,21 +355,16 @@ struct compressed_bio *btrfs_alloc_compressed_write(struct btrfs_inode *inode, } /* - * Add extra pages in the same compressed file extent so that we don't need to + * Add extra folios in the same compressed file extent so that we don't need to * re-read the same extent again and again. * - * NOTE: this won't work well for subpage, as for subpage read, we lock the - * full page then submit bio for each compressed/regular extents. - * - * This means, if we have several sectors in the same page points to the same - * on-disk compressed data, we will re-read the same extent many times and - * this function can only help for the next page. + * If in the same folio, we have several non-contiguous blocks which are pointing + * to the same on-disk compressed data, we will re-read the same extent many + * times, as this function can only help cross folio situations. */ -static noinline int add_ra_bio_pages(struct inode *inode, - u64 compressed_end, - struct compressed_bio *cb, - int *memstall, unsigned long *pflags, - bool direct_reclaim) +static noinline int add_ra_bio_folios(struct inode *inode, u64 compressed_end, + struct compressed_bio *cb, int *memstall, + unsigned long *pflags, bool direct_reclaim) { struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); pgoff_t end_index; @@ -391,16 +386,6 @@ static noinline int add_ra_bio_pages(struct inode *inode, if (isize == 0) return 0; - /* - * For current subpage support, we only support 64K page size, - * which means maximum compressed extent size (128K) is just 2x page - * size. - * This makes readahead less effective, so here disable readahead for - * subpage for now, until full compressed write is supported. - */ - if (fs_info->sectorsize < PAGE_SIZE) - return 0; - /* For bs > ps cases, we don't support readahead for compressed folios for now. */ if (fs_info->block_min_order) return 0; @@ -416,7 +401,7 @@ static noinline int add_ra_bio_pages(struct inode *inode, } while (cur < compressed_end) { - pgoff_t page_end; + u64 folio_end; pgoff_t pg_index = cur >> PAGE_SHIFT; gfp_t masked_constraint_gfp; u32 add_size; @@ -438,8 +423,8 @@ static noinline int add_ra_bio_pages(struct inode *inode, break; /* - * Jump to next page start as we already have page for - * current offset. + * Jump to the next folio as we already have a folio for + * the current offset. */ cur += (folio_sz - offset); continue; @@ -457,8 +442,8 @@ static noinline int add_ra_bio_pages(struct inode *inode, break; if (filemap_add_folio(mapping, folio, pg_index, cache_gfp)) { - /* There is already a page, skip to page end */ - cur += folio_size(folio); + /* There is already a folio, skip to the folio end. */ + cur += folio_size(folio) - offset_in_folio(folio, cur); folio_put(folio); continue; } @@ -475,14 +460,14 @@ static noinline int add_ra_bio_pages(struct inode *inode, break; } - page_end = (pg_index << PAGE_SHIFT) + folio_size(folio) - 1; - btrfs_lock_extent(tree, cur, page_end, NULL); + folio_end = folio_next_pos(folio) - 1; + btrfs_lock_extent(tree, cur, folio_end, NULL); read_lock(&em_tree->lock); - em = btrfs_lookup_extent_mapping(em_tree, cur, page_end + 1 - cur); + em = btrfs_lookup_extent_mapping(em_tree, cur, folio_end + 1 - cur); read_unlock(&em_tree->lock); /* - * At this point, we have a locked page in the page cache for + * At this point, we have a locked folio in the page cache for * these bytes in the file. But, we have to make sure they map * to this compressed extent on disk. */ @@ -491,14 +476,14 @@ static noinline int add_ra_bio_pages(struct inode *inode, (btrfs_extent_map_block_start(em) >> SECTOR_SHIFT) != orig_bio->bi_iter.bi_sector) { btrfs_free_extent_map(em); - btrfs_unlock_extent(tree, cur, page_end, NULL); + btrfs_unlock_extent(tree, cur, folio_end, NULL); folio_unlock(folio); folio_put(folio); break; } - add_size = min(btrfs_extent_map_end(em), page_end + 1) - cur; + add_size = min(btrfs_extent_map_end(em), folio_end + 1) - cur; btrfs_free_extent_map(em); - btrfs_unlock_extent(tree, cur, page_end, NULL); + btrfs_unlock_extent(tree, cur, folio_end, NULL); if (folio_contains(folio, end_index)) { size_t zero_offset = offset_in_folio(folio, isize); @@ -516,13 +501,7 @@ static noinline int add_ra_bio_pages(struct inode *inode, folio_put(folio); break; } - /* - * If it's subpage, we also need to increase its - * subpage::readers number, as at endio we will decrease - * subpage::readers and to unlock the page. - */ - if (fs_info->sectorsize < PAGE_SIZE) - btrfs_folio_set_lock(fs_info, folio, cur, add_size); + btrfs_folio_set_lock(fs_info, folio, cur, add_size); folio_put(folio); cur += add_size; } @@ -613,8 +592,8 @@ void btrfs_submit_compressed_read(struct btrfs_bio *bbio) } ASSERT(cb->bbio.bio.bi_iter.bi_size == compressed_len); - add_ra_bio_pages(&inode->vfs_inode, em_start + em_len, cb, &memstall, - &pflags, !(bbio->bio.bi_opf & REQ_RAHEAD)); + add_ra_bio_folios(&inode->vfs_inode, em_start + em_len, cb, &memstall, + &pflags, !(bbio->bio.bi_opf & REQ_RAHEAD)); cb->len = bbio->bio.bi_iter.bi_size; cb->bbio.bio.bi_iter.bi_sector = bbio->bio.bi_iter.bi_sector; @@ -1191,22 +1170,6 @@ void __cold btrfs_exit_compress(void) bioset_exit(&btrfs_compressed_bioset); } -/* - * The bvec is a single page bvec from a bio that contains folios from a filemap. - * - * Since the folio may be a large one, and if the bv_page is not a head page of - * a large folio, then page->index is unreliable. - * - * Thus we need this helper to grab the proper file offset. - */ -static u64 file_offset_from_bvec(const struct bio_vec *bvec) -{ - const struct page *page = bvec->bv_page; - const struct folio *folio = page_folio(page); - - return (page_pgoff(folio, page) << PAGE_SHIFT) + bvec->bv_offset; -} - /* * Copy decompressed data from working buffer to pages. * @@ -1259,7 +1222,7 @@ int btrfs_decompress_buf2page(const char *buf, u32 buf_len, * cb->start may underflow, but subtracting that value can still * give us correct offset inside the full decompressed extent. */ - bvec_offset = file_offset_from_bvec(&bvec) - cb->start; + bvec_offset = page_offset(bvec.bv_page) + bvec.bv_offset - cb->start; /* Haven't reached the bvec range, exit */ if (decompressed + buf_len <= bvec_offset) diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c index d70da290bedf..49fb6b816aa9 100644 --- a/fs/btrfs/ctree.c +++ b/fs/btrfs/ctree.c @@ -475,13 +475,10 @@ int btrfs_force_cow_block(struct btrfs_trans_handle *trans, struct extent_buffer *cow; int level, ret; int last_ref = 0; - int unlock_orig = 0; + const bool unlock_orig = (*cow_ret == buf); u64 parent_start = 0; u64 reloc_src_root = 0; - if (*cow_ret == buf) - unlock_orig = 1; - btrfs_assert_tree_write_locked(buf); WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && @@ -1497,17 +1494,11 @@ read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, if (p->reada == READA_FORWARD_ALWAYS) reada_for_search(fs_info, p, parent_level, slot, key->objectid); - /* first we do an atomic uptodate check */ - if (btrfs_buffer_uptodate(tmp, check.transid, NULL) > 0) { - /* - * Do extra check for first_key, eb can be stale due to - * being cached, read from scrub, or have multiple - * parents (shared tree blocks). - */ - if (unlikely(btrfs_verify_level_key(tmp, &check))) { - ret = -EUCLEAN; - goto out; - } + /* Check if the cached eb is uptodate. */ + ret = btrfs_buffer_uptodate(tmp, check.transid, &check); + if (unlikely(ret < 0)) + goto out; + if (ret > 0) { *eb_ret = tmp; tmp = NULL; ret = 0; @@ -2075,7 +2066,7 @@ again: } while (b) { - int dec = 0; + bool dec = false; int ret2; level = btrfs_header_level(b); @@ -2158,7 +2149,7 @@ cow_done: prev_cmp = ret; if (ret && slot > 0) { - dec = 1; + dec = true; slot--; } p->slots[level] = slot; @@ -2288,7 +2279,7 @@ again: p->locks[level] = BTRFS_READ_LOCK; while (b) { - int dec = 0; + bool dec = false; int ret2; level = btrfs_header_level(b); @@ -2313,7 +2304,7 @@ again: } if (ret && slot > 0) { - dec = 1; + dec = true; slot--; } p->slots[level] = slot; @@ -3674,7 +3665,7 @@ static noinline int split_leaf(struct btrfs_trans_handle *trans, int wret; int split; int num_doubles = 0; - int tried_avoid_double = 0; + bool tried_avoid_double = false; l = path->nodes[0]; slot = path->slots[0]; @@ -3836,7 +3827,7 @@ again: push_for_double: push_for_double_split(trans, root, path, data_size); - tried_avoid_double = 1; + tried_avoid_double = true; if (btrfs_leaf_free_space(path->nodes[0]) >= data_size) return 0; goto again; diff --git a/fs/btrfs/defrag.c b/fs/btrfs/defrag.c index 7e2db5d3a4d4..f0c6758b7055 100644 --- a/fs/btrfs/defrag.c +++ b/fs/btrfs/defrag.c @@ -859,23 +859,6 @@ again: if (IS_ERR(folio)) return folio; - /* - * Since we can defragment files opened read-only, we can encounter - * transparent huge pages here (see CONFIG_READ_ONLY_THP_FOR_FS). - * - * The IO for such large folios is not fully tested, thus return - * an error to reject such folios unless it's an experimental build. - * - * Filesystem transparent huge pages are typically only used for - * executables that explicitly enable them, so this isn't very - * restrictive. - */ - if (!IS_ENABLED(CONFIG_BTRFS_EXPERIMENTAL) && folio_test_large(folio)) { - folio_unlock(folio); - folio_put(folio); - return ERR_PTR(-ETXTBSY); - } - ret = set_folio_extent_mapped(folio); if (ret < 0) { folio_unlock(folio); @@ -1179,7 +1162,6 @@ static int defrag_one_locked_target(struct btrfs_inode *inode, if (start >= folio_next_pos(folio) || start + len <= folio_pos(folio)) continue; - btrfs_folio_clamp_clear_checked(fs_info, folio, start, len); btrfs_folio_clamp_set_dirty(fs_info, folio, start, len); } btrfs_delalloc_release_extents(inode, len); diff --git a/fs/btrfs/delalloc-space.c b/fs/btrfs/delalloc-space.c index 0970799d0aa4..d357ed7efd99 100644 --- a/fs/btrfs/delalloc-space.c +++ b/fs/btrfs/delalloc-space.c @@ -134,6 +134,8 @@ int btrfs_alloc_data_chunk_ondemand(const struct btrfs_inode *inode, u64 bytes) if (btrfs_is_free_space_inode(inode)) flush = BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE; + else if (btrfs_is_zoned(fs_info) && btrfs_is_data_reloc_root(root)) + flush = BTRFS_RESERVE_FLUSH_ZONED_RELOCATION; return btrfs_reserve_data_bytes(data_sinfo_for_inode(inode), bytes, flush); } @@ -279,7 +281,7 @@ static void btrfs_calculate_inode_block_rsv_size(struct btrfs_fs_info *fs_info, * * This is overestimating in most cases. */ - qgroup_rsv_size = (u64)outstanding_extents * fs_info->nodesize; + qgroup_rsv_size = ((u64)outstanding_extents << fs_info->nodesize_bits); spin_lock(&block_rsv->lock); block_rsv->size = reserve_size; @@ -309,7 +311,7 @@ static void calc_inode_reservations(struct btrfs_inode *inode, * for an inode update. */ *meta_reserve += inode_update; - *qgroup_reserve = nr_extents * fs_info->nodesize; + *qgroup_reserve = (nr_extents << fs_info->nodesize_bits); } int btrfs_delalloc_reserve_metadata(struct btrfs_inode *inode, u64 num_bytes, diff --git a/fs/btrfs/delayed-ref.c b/fs/btrfs/delayed-ref.c index 605858c2d9a9..8bc1929237f7 100644 --- a/fs/btrfs/delayed-ref.c +++ b/fs/btrfs/delayed-ref.c @@ -615,6 +615,9 @@ static bool insert_delayed_ref(struct btrfs_trans_handle *trans, struct btrfs_delayed_ref_node *exist; int mod; + ASSERT(ref->action == BTRFS_ADD_DELAYED_REF || + ref->action == BTRFS_DROP_DELAYED_REF); + spin_lock(&href->lock); exist = tree_insert(&href->ref_tree, ref); if (!exist) { @@ -641,7 +644,7 @@ static bool insert_delayed_ref(struct btrfs_trans_handle *trans, ASSERT(!list_empty(&exist->add_list)); list_del_init(&exist->add_list); } else { - ASSERT(0); + DEBUG_WARN(); } } else mod = -ref->ref_mod; diff --git a/fs/btrfs/dev-replace.c b/fs/btrfs/dev-replace.c index 8f8fa14886de..318ddb790429 100644 --- a/fs/btrfs/dev-replace.c +++ b/fs/btrfs/dev-replace.c @@ -307,6 +307,8 @@ static int btrfs_init_dev_replace_tgtdev(struct btrfs_fs_info *fs_info, device->bdev_file = bdev_file; set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state); set_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state); + /* Check the comment in btrfs_init_new_device() for the reason. */ + atomic_inc(&device->dev_stats_ccnt); device->dev_stats_valid = 1; set_blocksize(bdev_file, BTRFS_BDEV_BLOCKSIZE); device->fs_devices = fs_devices; @@ -1013,8 +1015,15 @@ error: /* write back the superblocks */ trans = btrfs_start_transaction(root, 0); - if (!IS_ERR(trans)) + if (!IS_ERR(trans)) { + /* + * Ignore any error here, if we failed to remove the DEV_STATS + * item for devid 0, it's not a big deal. We have other ways + * to address it. + */ + btrfs_remove_dev_stat_item(trans, BTRFS_DEV_REPLACE_DEVID); btrfs_commit_transaction(trans); + } mutex_unlock(&dev_replace->lock_finishing_cancel_unmount); diff --git a/fs/btrfs/direct-io.c b/fs/btrfs/direct-io.c index 57167d56dc72..460326d34143 100644 --- a/fs/btrfs/direct-io.c +++ b/fs/btrfs/direct-io.c @@ -15,10 +15,12 @@ struct btrfs_dio_data { ssize_t submitted; + loff_t old_isize; struct extent_changeset *data_reserved; struct btrfs_ordered_extent *ordered; bool data_space_reserved; bool nocow_done; + bool updated_isize; }; struct btrfs_dio_private { @@ -228,6 +230,7 @@ static int btrfs_get_blocks_direct_write(struct extent_map **map, bool space_reserved = false; u64 len = *lenp; u64 prev_len; + loff_t old_isize; int ret = 0; /* @@ -341,8 +344,14 @@ static int btrfs_get_blocks_direct_write(struct extent_map **map, * Need to update the i_size under the extent lock so buffered * readers will get the updated i_size when we unlock. */ - if (start + len > i_size_read(inode)) + old_isize = i_size_read(inode); + if (start + len > old_isize) { + if (!dio_data->updated_isize) { + dio_data->old_isize = old_isize; + dio_data->updated_isize = true; + } i_size_write(inode, start + len); + } out: if (ret && space_reserved) { btrfs_delalloc_release_extents(BTRFS_I(inode), len); @@ -624,12 +633,55 @@ static int btrfs_dio_iomap_end(struct inode *inode, loff_t pos, loff_t length, if (submitted < length) { pos += submitted; length -= submitted; - if (write) + if (write) { + /* + * Got a short write and have updated the isize, need to + * revert the isize change. + * + * Normally we need to update isize with extent lock hold, + * but we're safe due to the following factors: + * + * - Only a single writer can be enlarging isize + * Enlarging isize will take the exclusive inode lock. + * + * - Buffered readers need to wait for the OE we're holding + * Buffered readers will lock extent and wait for OE + * of the folio range, and since page cache is invalidated + * the OE wait can not be skipped. + * + * So here we are safe to revert the isize before + * finishing the OE, and no reader of the remaining range + * can see the enlarged size. + * + * TODO: Extend the DIO_LOCKED lifespan for direct writes, + * and only enlarge isize after a successful write. + */ + if (dio_data->updated_isize) { + u64 new_isize; + + if (submitted == 0) + new_isize = dio_data->old_isize; + else + new_isize = max(dio_data->old_isize, pos); + i_size_write(inode, new_isize); + dio_data->updated_isize = false; + } + /* + * We have a short write, if there is any range + * that is submitted properly, that part will have + * its own OE split from the original one. + * + * So for the OE at dio_data->ordered, it's the part + * that is not submitted, and should be marked + * as fully truncated. + */ + btrfs_mark_ordered_extent_truncated(dio_data->ordered, 0); btrfs_finish_ordered_extent(dio_data->ordered, - pos, length, false); - else + pos, length, true); + } else { btrfs_unlock_dio_extent(&BTRFS_I(inode)->io_tree, pos, pos + length - 1, NULL); + } ret = -ENOTBLK; } if (write) { @@ -926,7 +978,7 @@ again: if (ret > 0) written = ret; - if (iov_iter_count(from) > 0 && (ret == -EFAULT || ret > 0)) { + if (iov_iter_count(from) > 0 && (ret == -EFAULT || ret >= 0)) { const size_t left = iov_iter_count(from); /* * We have more data left to write. Try to fault in as many as diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index c0a30bb213d7..0a7d80da9c94 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -215,7 +215,7 @@ int btrfs_read_extent_buffer(struct extent_buffer *eb, const struct btrfs_tree_parent_check *check) { struct btrfs_fs_info *fs_info = eb->fs_info; - int failed = 0; + bool failed = false; int ret; int num_copies = 0; int mirror_num = 0; @@ -234,7 +234,7 @@ int btrfs_read_extent_buffer(struct extent_buffer *eb, break; if (!failed_mirror) { - failed = 1; + failed = true; failed_mirror = eb->read_mirror; } @@ -491,10 +491,34 @@ static bool btree_release_folio(struct folio *folio, gfp_t gfp_flags) static void btree_invalidate_folio(struct folio *folio, size_t offset, size_t length) { - struct extent_io_tree *tree; + struct extent_io_tree *tree = &folio_to_inode(folio)->io_tree; + struct extent_state *cached_state = NULL; + const u64 start = folio_pos(folio); + const u64 end = folio_next_pos(folio) - 1; + + /* + * The range must cover the full @folio. + * Btree inode is never exposed to regular file operations, thus there + * is no partial truncation. + * The folio is only invalidated when the btree inode is evicted. + */ + ASSERT(offset == 0, "folio=%llu offset=%zu", folio_pos(folio), offset); + ASSERT(length == folio_size(folio), "folio=%llu folio_size=%zu length=%zu", + folio_pos(folio), folio_size(folio), length); + + /* This function is only called for the btree inode */ + ASSERT(tree->owner == IO_TREE_BTREE_INODE_IO); + + btrfs_lock_extent(tree, start, end, &cached_state); + folio_wait_writeback(folio); + + /* + * Currently for btree io tree, only EXTENT_LOCKED is utilized, + * so here we only need to unlock the extent range to free any + * existing extent state. + */ + btrfs_unlock_extent(tree, start, end, &cached_state); - tree = &folio_to_inode(folio)->io_tree; - extent_invalidate_folio(tree, folio, offset); btree_release_folio(folio, GFP_NOFS); if (folio_get_private(folio)) { btrfs_warn(folio_to_fs_info(folio), @@ -539,7 +563,7 @@ static bool btree_dirty_folio(struct address_space *mapping, continue; } spin_unlock_irqrestore(&subpage->lock, flags); - cur = page_start + cur_bit * fs_info->sectorsize; + cur = page_start + (cur_bit << fs_info->sectorsize_bits); eb = find_extent_buffer(fs_info, cur); ASSERT(eb); @@ -1736,7 +1760,6 @@ static int read_backup_root(struct btrfs_fs_info *fs_info, u8 priority) /* helper to cleanup workers */ static void btrfs_stop_all_workers(struct btrfs_fs_info *fs_info) { - btrfs_destroy_workqueue(fs_info->fixup_workers); btrfs_destroy_workqueue(fs_info->delalloc_workers); btrfs_destroy_workqueue(fs_info->workers); if (fs_info->endio_workers) @@ -1944,9 +1967,6 @@ static int btrfs_init_workqueues(struct btrfs_fs_info *fs_info) fs_info->caching_workers = btrfs_alloc_workqueue(fs_info, "cache", flags, max_active, 0); - fs_info->fixup_workers = - btrfs_alloc_ordered_workqueue(fs_info, "fixup", ordered_flags); - fs_info->endio_workers = alloc_workqueue("btrfs-endio", flags, max_active); fs_info->endio_meta_workers = @@ -1972,7 +1992,7 @@ static int btrfs_init_workqueues(struct btrfs_fs_info *fs_info) fs_info->endio_workers && fs_info->endio_meta_workers && fs_info->endio_write_workers && fs_info->endio_freespace_worker && fs_info->rmw_workers && - fs_info->caching_workers && fs_info->fixup_workers && + fs_info->caching_workers && fs_info->delayed_workers && fs_info->qgroup_rescan_workers && fs_info->discard_ctl.discard_workers)) { return -ENOMEM; @@ -2776,6 +2796,7 @@ void btrfs_init_fs_info(struct btrfs_fs_info *fs_info) mutex_init(&fs_info->unused_bg_unpin_mutex); mutex_init(&fs_info->reclaim_bgs_lock); mutex_init(&fs_info->reloc_mutex); + spin_lock_init(&fs_info->reloc_ctl_lock); mutex_init(&fs_info->delalloc_root_mutex); mutex_init(&fs_info->zoned_meta_io_lock); mutex_init(&fs_info->zoned_data_reloc_io_lock); @@ -3276,6 +3297,64 @@ static bool fs_is_full_ro(const struct btrfs_fs_info *fs_info) return false; } +/* + * Try to wait for any metadata readahead, and invalidate all btree folios. + * + * If the invalidation failed, report any dirty/held extent buffers. + */ +static void invalidate_and_check_btree_folios(struct btrfs_fs_info *fs_info) +{ + unsigned long index = 0; + struct extent_buffer *eb; + int ret; + + ret = invalidate_inode_pages2(fs_info->btree_inode->i_mapping); + if (likely(ret == 0)) + return; + + /* + * Some btree pages can not be invalidated, this happens when some tree + * blocks are still held (either by readahead or some task is holding a ref). + */ + rcu_read_lock(); + xa_for_each(&fs_info->buffer_tree, index, eb) { + /* Increase the ref so that the eb won't disappear. */ + if (!refcount_inc_not_zero(&eb->refs)) + continue; + rcu_read_unlock(); + + /* Wait for any readahead first. */ + if (test_bit(EXTENT_BUFFER_READING, &eb->bflags)) + wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_READING, + TASK_UNINTERRUPTIBLE); + /* + * The refs threshold is 2, one held by us at the beginning + * of the loop, one for the ownership in the buffer tree. + */ + if (unlikely(refcount_read(&eb->refs) > 2 || extent_buffer_under_io(eb))) { + WARN_ON_ONCE(IS_ENABLED(CONFIG_BTRFS_DEBUG)); + btrfs_warn(fs_info, + "unable to release extent buffer %llu owner %llu gen %llu refs %u flags 0x%lx", + eb->start, btrfs_header_owner(eb), + btrfs_header_generation(eb), + refcount_read(&eb->refs), eb->bflags); + } + free_extent_buffer(eb); + rcu_read_lock(); + } + rcu_read_unlock(); + invalidate_inode_pages2(fs_info->btree_inode->i_mapping); +} + +static u32 calc_block_max_order(u32 sectorsize_bits) +{ + u32 max_size; + + max_size = min(BTRFS_MAX_BLOCKS_PER_FOLIO << sectorsize_bits, + BTRFS_MAX_FOLIO_SIZE); + return ilog2(round_up(max_size, PAGE_SIZE) >> PAGE_SHIFT); +} + int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_devices) { u32 sectorsize; @@ -3398,7 +3477,7 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device fs_info->sectorsize = sectorsize; fs_info->sectorsize_bits = ilog2(sectorsize); fs_info->block_min_order = ilog2(round_up(sectorsize, PAGE_SIZE) >> PAGE_SHIFT); - fs_info->block_max_order = ilog2((BITS_PER_LONG << fs_info->sectorsize_bits) >> PAGE_SHIFT); + fs_info->block_max_order = calc_block_max_order(fs_info->sectorsize_bits); fs_info->csums_per_leaf = BTRFS_MAX_ITEM_SIZE(fs_info) / fs_info->csum_size; fs_info->stripesize = stripesize; fs_info->fs_devices->fs_info = fs_info; @@ -3451,7 +3530,16 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device /* Update the values for the current filesystem. */ sb->s_blocksize = sectorsize; sb->s_blocksize_bits = blksize_bits(sectorsize); - memcpy(&sb->s_uuid, fs_info->fs_devices->fsid, BTRFS_FSID_SIZE); + /* + * When temp_fsid is active, fs_devices->fsid is assigned a random UUID + * at mount. This inconsistent UUID causes issues for layered filesystems + * like OverlayFS. Since metadata_uuid may or may not be set, provide the + * on-disk UUID directly from the super_copy. + */ + if (fs_info->fs_devices->temp_fsid) + memcpy(&sb->s_uuid, fs_info->super_copy->fsid, BTRFS_FSID_SIZE); + else + memcpy(&sb->s_uuid, fs_info->fs_devices->fsid, BTRFS_FSID_SIZE); mutex_lock(&fs_info->chunk_mutex); ret = btrfs_read_sys_array(fs_info); @@ -3591,6 +3679,13 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device } } + ret = btrfs_init_writeback_bio_size(fs_info); + if (ret) { + btrfs_err(fs_info, "failed to get optimum writeback size: %d", + ret); + goto fail_sysfs; + } + btrfs_free_zone_cache(fs_info); btrfs_check_active_zone_reservation(fs_info); @@ -3706,7 +3801,7 @@ fail_tree_roots: if (fs_info->data_reloc_root) btrfs_drop_and_free_fs_root(fs_info, fs_info->data_reloc_root); free_root_pointers(fs_info, true); - invalidate_inode_pages2(fs_info->btree_inode->i_mapping); + invalidate_and_check_btree_folios(fs_info); fail_sb_buffer: btrfs_stop_all_workers(fs_info); @@ -4209,7 +4304,6 @@ static void warn_about_uncommitted_trans(struct btrfs_fs_info *fs_info) list_del_init(&trans->list); btrfs_put_transaction(trans); - trace_btrfs_transaction_commit(fs_info); } ASSERT(!found); } @@ -4279,16 +4373,6 @@ void __cold close_ctree(struct btrfs_fs_info *fs_info) if (unlikely(BTRFS_FS_ERROR(fs_info))) btrfs_error_commit_super(fs_info); - /* - * Wait for any fixup workers to complete. - * If we don't wait for them here and they are still running by the time - * we call kthread_stop() against the cleaner kthread further below, we - * get an use-after-free on the cleaner because the fixup worker adds an - * inode to the list of delayed iputs and then attempts to wakeup the - * cleaner kthread, which was already stopped and destroyed. We parked - * already the cleaner, but below we run all pending delayed iputs. - */ - btrfs_flush_workqueue(fs_info->fixup_workers); /* * Similar case here, we have to wait for delalloc workers before we * proceed below and stop the cleaner kthread, otherwise we trigger a @@ -4412,7 +4496,7 @@ void __cold close_ctree(struct btrfs_fs_info *fs_info) ASSERT(list_empty(&fs_info->delayed_iputs)); set_bit(BTRFS_FS_CLOSING_DONE, &fs_info->flags); - if (btrfs_check_quota_leak(fs_info)) { + if (unlikely(btrfs_check_quota_leak(fs_info))) { DEBUG_WARN("qgroup reserved space leaked"); btrfs_err(fs_info, "qgroup reserved space leaked"); } @@ -4445,7 +4529,7 @@ void __cold close_ctree(struct btrfs_fs_info *fs_info) * We must make sure there is not any read request to * submit after we stop all workers. */ - invalidate_inode_pages2(fs_info->btree_inode->i_mapping); + invalidate_and_check_btree_folios(fs_info); btrfs_stop_all_workers(fs_info); /* @@ -4881,7 +4965,6 @@ static int btrfs_cleanup_transaction(struct btrfs_fs_info *fs_info) spin_unlock(&fs_info->trans_lock); btrfs_put_transaction(t); - trace_btrfs_transaction_commit(fs_info); spin_lock(&fs_info->trans_lock); } spin_unlock(&fs_info->trans_lock); diff --git a/fs/btrfs/extent-io-tree.c b/fs/btrfs/extent-io-tree.c index 626702244809..c18ea5ef2974 100644 --- a/fs/btrfs/extent-io-tree.c +++ b/fs/btrfs/extent-io-tree.c @@ -1763,7 +1763,7 @@ u64 btrfs_count_range_bits(struct extent_io_tree *tree, u64 cur_start = *start; u64 total_bytes = 0; u64 last = 0; - int found = 0; + bool found = false; if (WARN_ON(search_end < cur_start)) return 0; @@ -1817,7 +1817,7 @@ search: break; if (!found) { *start = max(cur_start, state->start); - found = 1; + found = true; } last = state->end; } else if (contig && found) { diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index 391fad41c3b6..624d76e0ca01 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -385,7 +385,7 @@ int btrfs_get_extent_inline_ref_type(const struct extent_buffer *eb, return type; } } else { - ASSERT(is_data == BTRFS_REF_TYPE_ANY); + ASSERT(is_data == BTRFS_REF_TYPE_ANY, "is_data=%d", is_data); return type; } } @@ -1699,13 +1699,13 @@ static int run_delayed_extent_op(struct btrfs_trans_handle *trans, struct extent_buffer *leaf; u32 item_size; int ret; - int metadata = 1; + bool metadata = true; if (TRANS_ABORTED(trans)) return 0; if (!btrfs_fs_incompat(fs_info, SKINNY_METADATA)) - metadata = 0; + metadata = false; path = btrfs_alloc_path(); if (!path) @@ -1745,7 +1745,7 @@ again: } if (ret > 0) { btrfs_release_path(path); - metadata = 0; + metadata = false; key.objectid = head->bytenr; key.type = BTRFS_EXTENT_ITEM_KEY; @@ -2108,7 +2108,8 @@ static noinline int __btrfs_run_delayed_refs(struct btrfs_trans_handle *trans, locked_ref = btrfs_select_ref_head(fs_info, delayed_refs); if (IS_ERR_OR_NULL(locked_ref)) { if (PTR_ERR(locked_ref) == -EAGAIN) { - continue; + count++; + goto again; } else { break; } @@ -2156,7 +2157,7 @@ static noinline int __btrfs_run_delayed_refs(struct btrfs_trans_handle *trans, * Either success case or btrfs_run_delayed_refs_for_head * returned -EAGAIN, meaning we need to select another head */ - +again: locked_ref = NULL; cond_resched(); } while ((min_bytes != U64_MAX && bytes_processed < min_bytes) || @@ -2531,8 +2532,11 @@ int btrfs_cross_ref_exist(struct btrfs_inode *inode, u64 offset, struct btrfs_key key; btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); - ASSERT(key.objectid == bytenr); - ASSERT(key.type == BTRFS_EXTENT_ITEM_KEY); + ASSERT(key.objectid == bytenr, + "key.objectid=%llu bytenr=%llu", + key.objectid, bytenr); + ASSERT(key.type == BTRFS_EXTENT_ITEM_KEY, "key.type=%u", + key.type); } } @@ -3279,7 +3283,7 @@ static int __btrfs_free_extent(struct btrfs_trans_handle *trans, int ret; int is_data; int extent_slot = 0; - int found_extent = 0; + bool found_extent = false; int num_to_del = 1; int refs_to_drop = node->ref_mod; u32 item_size; @@ -3335,12 +3339,12 @@ static int __btrfs_free_extent(struct btrfs_trans_handle *trans, break; if (key.type == BTRFS_EXTENT_ITEM_KEY && key.offset == num_bytes) { - found_extent = 1; + found_extent = true; break; } if (key.type == BTRFS_METADATA_ITEM_KEY && key.offset == owner_objectid) { - found_extent = 1; + found_extent = true; break; } @@ -4598,10 +4602,12 @@ static noinline int find_free_extent(struct btrfs_root *root, /* Use dedicated sub-space_info for dedicated block group users. */ if (ffe_ctl->for_data_reloc) { space_info = space_info->sub_group[0]; - ASSERT(space_info->subgroup_id == BTRFS_SUB_GROUP_DATA_RELOC); + ASSERT(space_info->subgroup_id == BTRFS_SUB_GROUP_DATA_RELOC, + "space_info->subgroup_id=%d", space_info->subgroup_id); } else if (ffe_ctl->for_treelog) { space_info = space_info->sub_group[0]; - ASSERT(space_info->subgroup_id == BTRFS_SUB_GROUP_TREELOG); + ASSERT(space_info->subgroup_id == BTRFS_SUB_GROUP_TREELOG, + "space_info->subgroup_id=%d", space_info->subgroup_id); } } if (!space_info) { @@ -5781,16 +5787,21 @@ static int check_next_block_uptodate(struct btrfs_trans_handle *trans, generation = btrfs_node_ptr_generation(path->nodes[level], path->slots[level]); - if (btrfs_buffer_uptodate(next, generation, NULL)) - return 0; - check.level = level - 1; check.transid = generation; check.owner_root = btrfs_root_id(root); check.has_first_key = true; btrfs_node_key_to_cpu(path->nodes[level], &check.first_key, path->slots[level]); + ret = btrfs_buffer_uptodate(next, generation, &check); + if (ret > 0) + return 0; btrfs_tree_unlock(next); + if (ret < 0) { + free_extent_buffer(next); + return ret; + } + if (level == 1) reada_walk_down(trans, root, wc, path); ret = btrfs_read_extent_buffer(next, &check); @@ -6613,12 +6624,16 @@ static int btrfs_trim_free_extents_throttle(struct btrfs_device *device, *trimmed = 0; - /* Discard not supported = nothing to do. */ - if (!bdev_max_discard_sectors(device->bdev)) + /* + * The caller only filters out MISSING devices, but a device that was + * missing at mount and later rescanned has MISSING cleared while bdev + * is still NULL and WRITEABLE is still unset. Skip those here. + */ + if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) || !device->bdev) return 0; - /* Not writable = nothing to do. */ - if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) + /* Discard not supported = nothing to do. */ + if (!bdev_max_discard_sectors(device->bdev)) return 0; /* No free space = nothing to do. */ @@ -6645,7 +6660,7 @@ static int btrfs_trim_free_extents_throttle(struct btrfs_device *device, start = max(start, cur_start); /* Check if there are any CHUNK_* bits left */ - if (start > device->total_bytes) { + if (unlikely(start > device->total_bytes)) { DEBUG_WARN(); btrfs_warn(fs_info, "ignoring attempt to trim beyond device size: offset %llu length %llu device %s device size %llu", diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 2275189b7860..7d604524e83c 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -130,7 +130,8 @@ struct btrfs_bio_ctrl { * extent_writepage_io(). * This is to avoid touching ranges covered by compression/inline. */ - unsigned long submit_bitmap; + unsigned long submit_bitmap[BITS_TO_LONGS(BTRFS_MAX_BLOCKS_PER_FOLIO)]; + struct readahead_control *ractl; /* @@ -250,8 +251,6 @@ static void process_one_folio(struct btrfs_fs_info *fs_info, ASSERT(end + 1 - start != 0 && end + 1 - start < U32_MAX); len = end + 1 - start; - if (page_ops & PAGE_SET_ORDERED) - btrfs_folio_clamp_set_ordered(fs_info, folio, start, len); if (page_ops & PAGE_START_WRITEBACK) { btrfs_folio_clamp_clear_dirty(fs_info, folio, start, len); btrfs_folio_clamp_set_writeback(fs_info, folio, start, len); @@ -381,7 +380,7 @@ noinline_for_stack bool find_lock_delalloc_range(struct inode *inode, bool found; struct extent_state *cached_state = NULL; int ret; - int loops = 0; + bool loops = false; /* Caller should pass a valid @end to indicate the search range end */ ASSERT(orig_end > orig_start); @@ -438,7 +437,7 @@ again: cached_state = NULL; if (!loops) { max_bytes = fs_info->sectorsize; - loops = 1; + loops = true; goto again; } else { return false; @@ -530,8 +529,6 @@ static void end_bbio_data_write(struct btrfs_bio *bbio) u32 len = fi.length; bio_size += len; - ASSERT(btrfs_folio_test_ordered(fs_info, folio, start, len)); - btrfs_folio_clear_ordered(fs_info, folio, start, len); btrfs_folio_clear_writeback(fs_info, folio, start, len); } @@ -620,24 +617,24 @@ static void end_bbio_data_read(struct btrfs_bio *bbio) } /* - * Populate every free slot in a provided array with folios using GFP_NOFS. + * Populate every free slot in a provided array with folios. * - * @nr_folios: number of folios to allocate - * @order: the order of the folios to be allocated - * @folio_array: the array to fill with folios; any existing non-NULL entries in - * the array will be skipped + * @nr_folios: number of folios to allocate + * @order: folio order + * @folio_array: array to fill with folios; non-NULL entries are skipped + * @gfp: GFP flags for the allocation * * Return: 0 if all folios were able to be allocated; * -ENOMEM otherwise, the partially allocated folios would be freed and * the array slots zeroed */ int btrfs_alloc_folio_array(unsigned int nr_folios, unsigned int order, - struct folio **folio_array) + struct folio **folio_array, gfp_t gfp) { for (int i = 0; i < nr_folios; i++) { if (folio_array[i]) continue; - folio_array[i] = folio_alloc(GFP_NOFS, order); + folio_array[i] = folio_alloc(gfp, order); if (!folio_array[i]) goto error; } @@ -652,21 +649,18 @@ error: } /* - * Populate every free slot in a provided array with pages, using GFP_NOFS. + * Populate every free slot in a provided array with pages. * - * @nr_pages: number of pages to allocate - * @page_array: the array to fill with pages; any existing non-null entries in - * the array will be skipped - * @nofail: whether using __GFP_NOFAIL flag + * @nr_pages: number of pages to allocate + * @page_array: array to fill; non-NULL entries are skipped + * @gfp: GFP flags for the allocation * * Return: 0 if all pages were able to be allocated; * -ENOMEM otherwise, the partially allocated pages would be freed and * the array slots zeroed */ -int btrfs_alloc_page_array(unsigned int nr_pages, struct page **page_array, - bool nofail) +int btrfs_alloc_page_array(unsigned int nr_pages, struct page **page_array, gfp_t gfp) { - const gfp_t gfp = nofail ? (GFP_NOFS | __GFP_NOFAIL) : GFP_NOFS; unsigned int allocated; for (allocated = 0; allocated < nr_pages;) { @@ -690,13 +684,13 @@ int btrfs_alloc_page_array(unsigned int nr_pages, struct page **page_array, * * For now, the folios populated are always in order 0 (aka, single page). */ -static int alloc_eb_folio_array(struct extent_buffer *eb, bool nofail) +static int alloc_eb_folio_array(struct extent_buffer *eb, gfp_t gfp) { struct page *page_array[INLINE_EXTENT_BUFFER_PAGES] = { 0 }; int num_pages = num_extent_pages(eb); int ret; - ret = btrfs_alloc_page_array(num_pages, page_array, nofail); + ret = btrfs_alloc_page_array(num_pages, page_array, gfp); if (ret < 0) return ret; @@ -729,9 +723,9 @@ static bool btrfs_bio_is_contig(struct btrfs_bio_ctrl *bio_ctrl, bio_end_sector(bio) == sector; } -static void alloc_new_bio(struct btrfs_inode *inode, - struct btrfs_bio_ctrl *bio_ctrl, - u64 disk_bytenr, u64 file_offset) +static int alloc_new_bio(struct btrfs_inode *inode, + struct btrfs_bio_ctrl *bio_ctrl, + u64 disk_bytenr, u64 file_offset) { struct btrfs_fs_info *fs_info = inode->root->fs_info; struct btrfs_bio *bbio; @@ -748,13 +742,25 @@ static void alloc_new_bio(struct btrfs_inode *inode, if (bio_ctrl->wbc) { struct btrfs_ordered_extent *ordered; + /* This must be a write for data inodes. */ + ASSERT(btrfs_op(&bio_ctrl->bbio->bio) == BTRFS_MAP_WRITE); + ASSERT(is_data_inode(inode)); + ordered = btrfs_lookup_ordered_extent(inode, file_offset); - if (ordered) { - bio_ctrl->len_to_oe_boundary = min_t(u32, U32_MAX, - ordered->file_offset + - ordered->disk_num_bytes - file_offset); - bbio->ordered = ordered; + if (unlikely(!ordered)) { + bio_ctrl->bbio = NULL; + bio_ctrl->next_file_offset = 0; + bio_put(&bbio->bio); + btrfs_err_rl(fs_info, + "root %lld ino %llu file offset %llu is marked dirty without notifying the fs", + btrfs_root_id(inode->root), btrfs_ino(inode), + file_offset); + return -EUCLEAN; } + bio_ctrl->len_to_oe_boundary = min_t(u32, U32_MAX, + ordered->file_offset + + ordered->disk_num_bytes - file_offset); + bbio->ordered = ordered; /* * Pick the last added device to support cgroup writeback. For @@ -765,6 +771,7 @@ static void alloc_new_bio(struct btrfs_inode *inode, bio_set_dev(&bbio->bio, fs_info->fs_devices->latest_dev->bdev); wbc_init_bio(bio_ctrl->wbc, &bbio->bio); } + return 0; } /* @@ -780,14 +787,19 @@ static void alloc_new_bio(struct btrfs_inode *inode, * new one in @bio_ctrl->bbio. * The mirror number for this IO should already be initialized in * @bio_ctrl->mirror_num. + * + * Return the number of bytes that are queued into a bio. + * If the returned bytes is smaller than @size, it means we hit a critical error + * for data write, where there is no ordered extent for the range. */ -static void submit_extent_folio(struct btrfs_bio_ctrl *bio_ctrl, - u64 disk_bytenr, struct folio *folio, - size_t size, unsigned long pg_offset, - u64 read_em_generation) +static unsigned int submit_extent_folio(struct btrfs_bio_ctrl *bio_ctrl, + u64 disk_bytenr, struct folio *folio, + size_t size, unsigned long pg_offset, + u64 read_em_generation) { struct btrfs_inode *inode = folio_to_inode(folio); loff_t file_offset = folio_pos(folio) + pg_offset; + unsigned int queued = 0; ASSERT(pg_offset + size <= folio_size(folio)); ASSERT(bio_ctrl->end_io_func); @@ -800,8 +812,13 @@ static void submit_extent_folio(struct btrfs_bio_ctrl *bio_ctrl, u32 len = size; /* Allocate new bio if needed */ - if (!bio_ctrl->bbio) - alloc_new_bio(inode, bio_ctrl, disk_bytenr, file_offset); + if (!bio_ctrl->bbio) { + int ret; + + ret = alloc_new_bio(inode, bio_ctrl, disk_bytenr, file_offset); + if (ret < 0) + break; + } /* Cap to the current ordered extent boundary if there is one. */ if (len > bio_ctrl->len_to_oe_boundary) { @@ -829,6 +846,7 @@ static void submit_extent_folio(struct btrfs_bio_ctrl *bio_ctrl, pg_offset += len; disk_bytenr += len; file_offset += len; + queued += len; /* * len_to_oe_boundary defaults to U32_MAX, which isn't folio or @@ -857,7 +875,18 @@ static void submit_extent_folio(struct btrfs_bio_ctrl *bio_ctrl, /* Ordered extent boundary: move on to a new bio. */ if (bio_ctrl->len_to_oe_boundary == 0) submit_one_bio(bio_ctrl); + /* + * If we have accumulated decent amount of IO, send it to the + * block layer so that IO can run while we are accumulating + * more folios to write. + */ + else if (bio_ctrl->wbc && + bio_ctrl->bbio->bio.bi_iter.bi_size >= + inode->root->fs_info->writeback_bio_size) + submit_one_bio(bio_ctrl); + } while (size); + return queued; } static int attach_extent_buffer_folio(struct extent_buffer *eb, @@ -922,6 +951,17 @@ void clear_folio_extent_mapped(struct folio *folio) struct btrfs_fs_info *fs_info; ASSERT(folio->mapping); + /* + * The folio should not have writeback nor dirty flag set. + * + * If dirty flag is set, the folio can be written back again and we + * expect the private flag set for the folio. + * + * If writeback flag is set, the endio may need to utilize the + * private for btrfs_folio_state. + */ + ASSERT(!folio_test_dirty(folio)); + ASSERT(!folio_test_writeback(folio)); if (!folio_test_private(folio)) return; @@ -1030,6 +1070,7 @@ static int btrfs_do_readpage(struct folio *folio, struct extent_map **em_cached, u64 disk_bytenr; u64 block_start; u64 em_gen; + unsigned int queued; ASSERT(IS_ALIGNED(cur, fs_info->sectorsize)); if (cur >= last_byte) { @@ -1143,8 +1184,10 @@ static int btrfs_do_readpage(struct folio *folio, struct extent_map **em_cached, if (force_bio_submit) submit_one_bio(bio_ctrl); - submit_extent_folio(bio_ctrl, disk_bytenr, folio, blocksize, - pg_offset, em_gen); + queued = submit_extent_folio(bio_ctrl, disk_bytenr, folio, blocksize, + pg_offset, em_gen); + /* Read submission should not fail. */ + ASSERT(queued == blocksize); } return 0; } @@ -1422,7 +1465,7 @@ static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode, const u64 page_start = folio_pos(folio); const u64 page_end = page_start + folio_size(folio) - 1; const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio); - unsigned long delalloc_bitmap = 0; + unsigned long delalloc_bitmap[BITS_TO_LONGS(BTRFS_MAX_BLOCKS_PER_FOLIO)] = { 0 }; /* * Save the last found delalloc end. As the delalloc end can go beyond * page boundary, thus we cannot rely on subpage bitmap to locate the @@ -1447,14 +1490,9 @@ static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode, int ret = 0; /* Save the dirty bitmap as our submission bitmap will be a subset of it. */ - if (btrfs_is_subpage(fs_info, folio)) { - ASSERT(blocks_per_folio > 1); - btrfs_get_subpage_dirty_bitmap(fs_info, folio, &bio_ctrl->submit_bitmap); - } else { - bio_ctrl->submit_bitmap = 1; - } + btrfs_copy_subpage_dirty_bitmap(fs_info, folio, bio_ctrl->submit_bitmap); - for_each_set_bitrange(start_bit, end_bit, &bio_ctrl->submit_bitmap, + for_each_set_bitrange(start_bit, end_bit, bio_ctrl->submit_bitmap, blocks_per_folio) { u64 start = page_start + (start_bit << fs_info->sectorsize_bits); u32 len = (end_bit - start_bit) << fs_info->sectorsize_bits; @@ -1470,7 +1508,7 @@ static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode, delalloc_start = delalloc_end + 1; continue; } - set_delalloc_bitmap(folio, &delalloc_bitmap, delalloc_start, + set_delalloc_bitmap(folio, delalloc_bitmap, delalloc_start, min(delalloc_end, page_end) + 1 - delalloc_start); last_delalloc_end = delalloc_end; delalloc_start = delalloc_end + 1; @@ -1496,7 +1534,7 @@ static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode, found_len = last_delalloc_end + 1 - found_start; found = true; } else { - found = find_next_delalloc_bitmap(folio, &delalloc_bitmap, + found = find_next_delalloc_bitmap(folio, delalloc_bitmap, delalloc_start, &found_start, &found_len); } if (!found) @@ -1530,7 +1568,7 @@ static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode, btrfs_ino(inode), folio_pos(folio), blocks_per_folio, - &bio_ctrl->submit_bitmap, + bio_ctrl->submit_bitmap, found_start, found_len, ret); } else { /* @@ -1555,7 +1593,7 @@ static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode, fs_info->sectorsize_bits; unsigned int end_bit = (min(page_end + 1, found_start + found_len) - page_start) >> fs_info->sectorsize_bits; - bitmap_clear(&bio_ctrl->submit_bitmap, start_bit, end_bit - start_bit); + bitmap_clear(bio_ctrl->submit_bitmap, start_bit, end_bit - start_bit); } /* * Above btrfs_run_delalloc_range() may have unlocked the folio, @@ -1576,12 +1614,11 @@ static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode, fs_info->sectorsize_bits, blocks_per_folio); - for_each_set_bitrange(start_bit, end_bit, &bio_ctrl->submit_bitmap, + for_each_set_bitrange(start_bit, end_bit, bio_ctrl->submit_bitmap, bitmap_size) { u64 start = page_start + (start_bit << fs_info->sectorsize_bits); u32 len = (end_bit - start_bit) << fs_info->sectorsize_bits; - btrfs_folio_clear_ordered(fs_info, folio, start, len); btrfs_mark_ordered_io_finished(inode, start, len, false); } return ret; @@ -1602,7 +1639,7 @@ out: * If all ranges are submitted asynchronously, we just need to account * for them here. */ - if (bitmap_empty(&bio_ctrl->submit_bitmap, blocks_per_folio)) { + if (bitmap_empty(bio_ctrl->submit_bitmap, blocks_per_folio)) { wbc->nr_to_write -= delalloc_to_write; return 1; } @@ -1637,6 +1674,7 @@ static int submit_one_sector(struct btrfs_inode *inode, u64 extent_offset; u64 em_end; const u32 sectorsize = fs_info->sectorsize; + unsigned int queued; ASSERT(IS_ALIGNED(filepos, sectorsize)); @@ -1658,7 +1696,6 @@ static int submit_one_sector(struct btrfs_inode *inode, * ordered extent. */ btrfs_folio_clear_dirty(fs_info, folio, filepos, sectorsize); - btrfs_folio_clear_ordered(fs_info, folio, filepos, sectorsize); btrfs_folio_set_writeback(fs_info, folio, filepos, sectorsize); btrfs_folio_clear_writeback(fs_info, folio, filepos, sectorsize); @@ -1703,8 +1740,14 @@ static int submit_one_sector(struct btrfs_inode *inode, */ ASSERT(folio_test_writeback(folio)); - submit_extent_folio(bio_ctrl, disk_bytenr, folio, - sectorsize, filepos - folio_pos(folio), 0); + queued = submit_extent_folio(bio_ctrl, disk_bytenr, folio, + sectorsize, filepos - folio_pos(folio), 0); + if (unlikely(queued < sectorsize)) { + btrfs_folio_clear_writeback(fs_info, folio, filepos, sectorsize); + btrfs_mark_ordered_io_finished(inode, filepos, fs_info->sectorsize, + false); + return -EUCLEAN; + } return 0; } @@ -1723,7 +1766,6 @@ static noinline_for_stack int extent_writepage_io(struct btrfs_inode *inode, loff_t i_size) { struct btrfs_fs_info *fs_info = inode->root->fs_info; - unsigned long range_bitmap = 0; bool submitted_io = false; int found_error = 0; const u64 end = start + len; @@ -1738,28 +1780,18 @@ static noinline_for_stack int extent_writepage_io(struct btrfs_inode *inode, ASSERT(end <= folio_end, "start=%llu len=%u folio_start=%llu folio_size=%zu", start, len, folio_start, folio_size(folio)); - ret = btrfs_writepage_cow_fixup(folio); - if (ret == -EAGAIN) { - /* Fixup worker will requeue */ - folio_redirty_for_writepage(bio_ctrl->wbc, folio); - folio_unlock(folio); - return 1; - } - if (ret < 0) { - btrfs_folio_clear_dirty(fs_info, folio, start, len); - btrfs_folio_set_writeback(fs_info, folio, start, len); - btrfs_folio_clear_writeback(fs_info, folio, start, len); - return ret; - } - - bitmap_set(&range_bitmap, (start - folio_pos(folio)) >> fs_info->sectorsize_bits, - len >> fs_info->sectorsize_bits); - bitmap_and(&bio_ctrl->submit_bitmap, &bio_ctrl->submit_bitmap, &range_bitmap, - blocks_per_folio); + /* Truncate the submit bitmap to the current range. */ + if (start > folio_start) + bitmap_clear(bio_ctrl->submit_bitmap, 0, + (start - folio_start) >> fs_info->sectorsize_bits); + if (start + len < folio_end) + bitmap_clear(bio_ctrl->submit_bitmap, + (end - folio_start) >> fs_info->sectorsize_bits, + (folio_end - end) >> fs_info->sectorsize_bits); bio_ctrl->end_io_func = end_bbio_data_write; - for_each_set_bit(bit, &bio_ctrl->submit_bitmap, blocks_per_folio) { + for_each_set_bit(bit, bio_ctrl->submit_bitmap, blocks_per_folio) { cur = folio_pos(folio) + (bit << fs_info->sectorsize_bits); if (cur >= i_size) { @@ -1779,7 +1811,6 @@ static noinline_for_stack int extent_writepage_io(struct btrfs_inode *inode, spin_unlock(&inode->ordered_tree_lock); btrfs_put_ordered_extent(ordered); - btrfs_folio_clear_ordered(fs_info, folio, cur, fs_info->sectorsize); btrfs_mark_ordered_io_finished(inode, cur, fs_info->sectorsize, true); /* * This range is beyond i_size, thus we don't need to @@ -1819,6 +1850,25 @@ static noinline_for_stack int extent_writepage_io(struct btrfs_inode *inode, return found_error; } +static void bio_ctrl_init_submit_bitmap(struct btrfs_fs_info *fs_info, + struct folio *folio, + struct btrfs_bio_ctrl *bio_ctrl) +{ + const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio); + + ASSERT(blocks_per_folio <= BTRFS_MAX_BLOCKS_PER_FOLIO); + + /* + * Default to unlock the whole folio. + * The proper bitmap is not initialized until writepage_delalloc(). + * + * We're safe just to set the bitmap range [0, blocks_per_folio), as + * all later usage of the bitmap will follow the same range limit. + * Any bits beyond blocks_per_folio will be ignored. + */ + bitmap_set(bio_ctrl->submit_bitmap, 0, blocks_per_folio); +} + /* * the writepage semantics are similar to regular writepage. extent * records are inserted to lock ranges in the tree, and as dirty areas @@ -1853,12 +1903,7 @@ static int extent_writepage(struct folio *folio, struct btrfs_bio_ctrl *bio_ctrl if (folio_contains(folio, end_index)) folio_zero_range(folio, pg_offset, folio_size(folio) - pg_offset); - /* - * Default to unlock the whole folio. - * The proper bitmap can only be initialized until writepage_delalloc(). - */ - bio_ctrl->submit_bitmap = (unsigned long)-1; - + bio_ctrl_init_submit_bitmap(fs_info, folio, bio_ctrl); /* * If the page is dirty but without private set, it's marked dirty * without informing the fs. @@ -1867,13 +1912,9 @@ static int extent_writepage(struct folio *folio, struct btrfs_bio_ctrl *bio_ctrl * * So here we check if the page has private set to rule out such * case. - * But we also have a long history of relying on the COW fixup, - * so here we only enable this check for experimental builds until - * we're sure it's safe. */ - if (IS_ENABLED(CONFIG_BTRFS_EXPERIMENTAL) && - unlikely(!folio_test_private(folio))) { - WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG)); + if (unlikely(!folio_test_private(folio))) { + DEBUG_WARN(); btrfs_err_rl(fs_info, "root %lld ino %llu folio %llu is marked dirty without notifying the fs", btrfs_root_id(inode->root), @@ -1901,7 +1942,7 @@ static int extent_writepage(struct folio *folio, struct btrfs_bio_ctrl *bio_ctrl "failed to submit blocks, root=%lld inode=%llu folio=%llu submit_bitmap=%*pbl: %d", btrfs_root_id(inode->root), btrfs_ino(inode), folio_pos(folio), blocks_per_folio, - &bio_ctrl->submit_bitmap, ret); + bio_ctrl->submit_bitmap, ret); bio_ctrl->wbc->nr_to_write--; @@ -2318,13 +2359,13 @@ int btree_writepages(struct address_space *mapping, struct writeback_control *wb struct btrfs_eb_write_context ctx = { .wbc = wbc }; struct btrfs_fs_info *fs_info = inode_to_fs_info(mapping->host); int ret = 0; - int done = 0; + bool done = false; int nr_to_write_done = 0; struct eb_batch batch; unsigned int nr_ebs; unsigned long index; unsigned long end; - int scanned = 0; + bool scanned = false; xa_mark_t tag; eb_batch_init(&batch); @@ -2341,7 +2382,7 @@ int btree_writepages(struct address_space *mapping, struct writeback_control *wb index = (wbc->range_start >> fs_info->nodesize_bits); end = (wbc->range_end >> fs_info->nodesize_bits); - scanned = 1; + scanned = true; } if (wbc->sync_mode == WB_SYNC_ALL) tag = PAGECACHE_TAG_TOWRITE; @@ -2364,7 +2405,7 @@ retry: ret = 0; if (ret) { - done = 1; + done = true; break; } continue; @@ -2390,7 +2431,7 @@ retry: * We hit the last page and there is more work to be done: wrap * back to the start of the file */ - scanned = 1; + scanned = true; index = 0; goto retry; } @@ -2430,15 +2471,15 @@ static int extent_write_cache_pages(struct address_space *mapping, struct writeback_control *wbc = bio_ctrl->wbc; struct inode *inode = mapping->host; int ret = 0; - int done = 0; + bool done = false; int nr_to_write_done = 0; struct folio_batch fbatch; unsigned int nr_folios; pgoff_t index; pgoff_t end; /* Inclusive */ pgoff_t done_index; - int range_whole = 0; - int scanned = 0; + bool range_whole = false; + bool scanned = false; xa_mark_t tag; /* @@ -2466,8 +2507,8 @@ static int extent_write_cache_pages(struct address_space *mapping, index = wbc->range_start >> PAGE_SHIFT; end = wbc->range_end >> PAGE_SHIFT; if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) - range_whole = 1; - scanned = 1; + range_whole = true; + scanned = true; } /* @@ -2544,14 +2585,14 @@ retry: } if (folio_test_writeback(folio) || - !folio_clear_dirty_for_io(folio)) { + !folio_test_dirty(folio)) { folio_unlock(folio); continue; } ret = extent_writepage(folio, bio_ctrl); if (ret < 0) { - done = 1; + done = true; break; } @@ -2571,7 +2612,7 @@ retry: * We hit the last page and there is more work to be done: wrap * back to the start of the file */ - scanned = 1; + scanned = true; index = 0; /* @@ -2648,7 +2689,7 @@ void extent_write_locked_range(struct inode *inode, const struct folio *locked_f * Set the submission bitmap to submit all sectors. * extent_writepage_io() will do the truncation correctly. */ - bio_ctrl.submit_bitmap = (unsigned long)-1; + bio_ctrl_init_submit_bitmap(fs_info, folio, &bio_ctrl); ret = extent_writepage_io(BTRFS_I(inode), folio, cur, cur_len, &bio_ctrl, i_size); if (ret == 1) @@ -2716,38 +2757,6 @@ void btrfs_readahead(struct readahead_control *rac) submit_one_bio(&bio_ctrl); } -/* - * basic invalidate_folio code, this waits on any locked or writeback - * ranges corresponding to the folio, and then deletes any extent state - * records from the tree - */ -int extent_invalidate_folio(struct extent_io_tree *tree, - struct folio *folio, size_t offset) -{ - struct extent_state *cached_state = NULL; - u64 start = folio_pos(folio); - u64 end = start + folio_size(folio) - 1; - size_t blocksize = folio_to_fs_info(folio)->sectorsize; - - /* This function is only called for the btree inode */ - ASSERT(tree->owner == IO_TREE_BTREE_INODE_IO); - - start += ALIGN(offset, blocksize); - if (start > end) - return 0; - - btrfs_lock_extent(tree, start, end, &cached_state); - folio_wait_writeback(folio); - - /* - * Currently for btree io tree, only EXTENT_LOCKED is utilized, - * so here we only need to unlock the extent range to free any - * existing extent state. - */ - btrfs_unlock_extent(tree, start, end, &cached_state); - return 0; -} - /* * A helper for struct address_space_operations::release_folio, this tests for * areas of the folio that are locked or under IO and drops the related state @@ -2877,12 +2886,6 @@ next: return try_release_extent_state(io_tree, folio); } -static int extent_buffer_under_io(const struct extent_buffer *eb) -{ - return (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) || - test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)); -} - static bool folio_range_has_eb(struct folio *folio) { struct btrfs_folio_state *bfs; @@ -3097,7 +3100,7 @@ struct extent_buffer *btrfs_clone_extent_buffer(const struct extent_buffer *src) */ set_bit(EXTENT_BUFFER_UNMAPPED, &new->bflags); - ret = alloc_eb_folio_array(new, false); + ret = alloc_eb_folio_array(new, GFP_NOFS); if (ret) goto release_eb; @@ -3138,7 +3141,7 @@ struct extent_buffer *alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info, if (!eb) return NULL; - ret = alloc_eb_folio_array(eb, false); + ret = alloc_eb_folio_array(eb, GFP_NOFS); if (ret) goto release_eb; @@ -3394,8 +3397,8 @@ retry: finish: spin_lock(&mapping->i_private_lock); if (existing_folio && btrfs_meta_is_subpage(fs_info)) { - /* We're going to reuse the existing page, can drop our folio now. */ - __free_page(folio_page(eb->folios[i], 0)); + /* We're going to reuse the existing folio, can drop our folio now. */ + folio_put(eb->folios[i]); eb->folios[i] = existing_folio; } else if (existing_folio) { struct extent_buffer *existing_eb; @@ -3410,7 +3413,7 @@ finish: return 1; } /* The extent buffer no longer exists, we can reuse the folio. */ - __free_page(folio_page(eb->folios[i], 0)); + folio_put(eb->folios[i]); eb->folios[i] = existing_folio; } eb->folio_size = folio_size(eb->folios[i]); @@ -3441,7 +3444,7 @@ struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info, struct btrfs_folio_state *prealloc = NULL; u64 lockdep_owner = owner_root; bool page_contig = true; - int uptodate = 1; + bool uptodate = true; int ret; if (check_eb_alignment(fs_info, start)) @@ -3491,8 +3494,12 @@ struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info, } reallocate: - /* Allocate all pages first. */ - ret = alloc_eb_folio_array(eb, true); + /* + * Allocate all pages first. These will be attached to btree_inode->i_mapping + * below (added to LRU, served by btree_migrate_folio), so request + * __GFP_MOVABLE so the page allocator places them in MOVABLE pageblocks. + */ + ret = alloc_eb_folio_array(eb, GFP_NOFS | __GFP_NOFAIL | __GFP_MOVABLE); if (ret < 0) { btrfs_free_folio_state(prealloc); goto out; @@ -3551,7 +3558,7 @@ reallocate: page_contig = false; if (!btrfs_meta_folio_test_uptodate(folio, eb)) - uptodate = 0; + uptodate = false; /* * We can't unlock the pages just yet since the extent buffer @@ -3739,17 +3746,6 @@ void free_extent_buffer_stale(struct extent_buffer *eb) release_extent_buffer(eb); } -static void btree_clear_folio_dirty_tag(struct folio *folio) -{ - ASSERT(!folio_test_dirty(folio)); - ASSERT(folio_test_locked(folio)); - xa_lock_irq(&folio->mapping->i_pages); - if (!folio_test_dirty(folio)) - __xa_clear_mark(&folio->mapping->i_pages, folio->index, - PAGECACHE_TAG_DIRTY); - xa_unlock_irq(&folio->mapping->i_pages); -} - void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans, struct extent_buffer *eb) { @@ -3790,7 +3786,7 @@ void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans, folio_lock(folio); last = btrfs_meta_folio_clear_and_test_dirty(folio, eb); if (last) - btree_clear_folio_dirty_tag(folio); + btrfs_clear_folio_dirty_tag(folio); folio_unlock(folio); } WARN_ON(refcount_read(&eb->refs) == 0); @@ -3981,15 +3977,14 @@ int read_extent_buffer_pages(struct extent_buffer *eb, int mirror_num, return 0; } -static bool report_eb_range(const struct extent_buffer *eb, unsigned long start, - unsigned long len) +/* Never inlined to decrease code size, as this is called in a cold path. */ +static noinline void report_eb_range(const struct extent_buffer *eb, + unsigned long start, unsigned long len) { btrfs_warn(eb->fs_info, "access to eb bytenr %llu len %u out of range start %lu len %lu", eb->start, eb->len, start, len); DEBUG_WARN(); - - return true; } /* @@ -3999,14 +3994,16 @@ static bool report_eb_range(const struct extent_buffer *eb, unsigned long start, * * Caller should not touch the dst/src memory if this function returns error. */ -static inline int check_eb_range(const struct extent_buffer *eb, - unsigned long start, unsigned long len) +static inline bool check_eb_range(const struct extent_buffer *eb, + unsigned long start, unsigned long len) { unsigned long offset; /* start, start + len should not go beyond eb->len nor overflow */ - if (unlikely(check_add_overflow(start, len, &offset) || offset > eb->len)) - return report_eb_range(eb, start, len); + if (unlikely(check_add_overflow(start, len, &offset) || offset > eb->len)) { + report_eb_range(eb, start, len); + return true; + } return false; } @@ -4660,7 +4657,7 @@ void btrfs_readahead_tree_block(struct btrfs_fs_info *fs_info, if (IS_ERR(eb)) return; - if (btrfs_buffer_uptodate(eb, gen, NULL)) { + if (btrfs_buffer_uptodate(eb, gen, &check)) { free_extent_buffer(eb); return; } diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h index b310a5145cf6..9896e15ddc40 100644 --- a/fs/btrfs/extent_io.h +++ b/fs/btrfs/extent_io.h @@ -55,7 +55,6 @@ enum { /* Page starts writeback, clear dirty bit and set writeback bit */ ENUM_BIT(PAGE_START_WRITEBACK), ENUM_BIT(PAGE_END_WRITEBACK), - ENUM_BIT(PAGE_SET_ORDERED), }; /* @@ -327,6 +326,12 @@ static inline bool extent_buffer_uptodate(const struct extent_buffer *eb) return test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); } +static inline bool extent_buffer_under_io(const struct extent_buffer *eb) +{ + return (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) || + test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)); +} + int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv, unsigned long start, unsigned long len); void read_extent_buffer(const struct extent_buffer *eb, void *dst, @@ -381,15 +386,23 @@ void extent_clear_unlock_delalloc(struct btrfs_inode *inode, u64 start, u64 end, const struct folio *locked_folio, struct extent_state **cached, u32 bits_to_clear, unsigned long page_ops); -int extent_invalidate_folio(struct extent_io_tree *tree, - struct folio *folio, size_t offset); void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans, struct extent_buffer *buf); -int btrfs_alloc_page_array(unsigned int nr_pages, struct page **page_array, - bool nofail); +static inline void btrfs_clear_folio_dirty_tag(struct folio *folio) +{ + ASSERT(!folio_test_dirty(folio)); + ASSERT(folio_test_locked(folio)); + ASSERT(folio->mapping); + xa_lock_irq(&folio->mapping->i_pages); + __xa_clear_mark(&folio->mapping->i_pages, folio->index, + PAGECACHE_TAG_DIRTY); + xa_unlock_irq(&folio->mapping->i_pages); +} + +int btrfs_alloc_page_array(unsigned int nr_pages, struct page **page_array, gfp_t gfp); int btrfs_alloc_folio_array(unsigned int nr_folios, unsigned int order, - struct folio **folio_array); + struct folio **folio_array, gfp_t gfp); #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS bool find_lock_delalloc_range(struct inode *inode, diff --git a/fs/btrfs/extent_map.c b/fs/btrfs/extent_map.c index 6b79bff241f2..fce9c5cc0122 100644 --- a/fs/btrfs/extent_map.c +++ b/fs/btrfs/extent_map.c @@ -717,7 +717,7 @@ int btrfs_add_extent_mapping(struct btrfs_inode *inode, * file offset. Here just do a sanity check. */ if (em->disk_bytenr == EXTENT_MAP_INLINE) - ASSERT(em->start == 0); + ASSERT(em->start == 0, "em->start=%llu", em->start); ret = add_extent_mapping(inode, em, false); /* it is possible that someone inserted the extent into the tree @@ -761,7 +761,7 @@ int btrfs_add_extent_mapping(struct btrfs_inode *inode, } } - ASSERT(ret == 0 || ret == -EEXIST); + ASSERT(ret == 0 || ret == -EEXIST, "ret=%d", ret); return ret; } @@ -943,7 +943,7 @@ void btrfs_drop_extent_map_range(struct btrfs_inode *inode, u64 start, u64 end, ret = add_extent_mapping(inode, split, modified); /* Logic error, shouldn't happen. */ - ASSERT(ret == 0); + ASSERT(ret == 0, "ret=%d", ret); if (WARN_ON(ret != 0) && modified) btrfs_set_inode_full_sync(inode); } diff --git a/fs/btrfs/fiemap.c b/fs/btrfs/fiemap.c index 27d361c7adc4..6263e837093e 100644 --- a/fs/btrfs/fiemap.c +++ b/fs/btrfs/fiemap.c @@ -112,7 +112,7 @@ static int emit_fiemap_extent(struct fiemap_extent_info *fieinfo, u64 cache_end; /* Set at the end of extent_fiemap(). */ - ASSERT((flags & FIEMAP_EXTENT_LAST) == 0); + ASSERT((flags & FIEMAP_EXTENT_LAST) == 0, "flags=0x%u", flags); if (!cache->cached) goto assign; diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c index d72249390030..9f6454e9db81 100644 --- a/fs/btrfs/file-item.c +++ b/fs/btrfs/file-item.c @@ -325,7 +325,9 @@ static int search_csum_tree(struct btrfs_fs_info *fs_info, csum_start = key.offset; csum_len = (itemsize / csum_size) * sectorsize; - ASSERT(in_range(disk_bytenr, csum_start, csum_len)); + ASSERT(in_range(disk_bytenr, csum_start, csum_len), + "disk_bytenr=%llu csum_start=%llu csum_len=%llu", + disk_bytenr, csum_start, csum_len); found: ret = (min(csum_start + csum_len, disk_bytenr + len) - @@ -1307,7 +1309,7 @@ found: index += ins_size; ins_size /= csum_size; - total_bytes += ins_size * fs_info->sectorsize; + total_bytes += (ins_size << fs_info->sectorsize_bits); if (total_bytes < sums->len) { btrfs_release_path(path); diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index 8c171ed07008..a2a2df2df786 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -49,14 +49,6 @@ static void btrfs_drop_folio(struct btrfs_fs_info *fs_info, struct folio *folio, u64 block_len = round_up(pos + copied, fs_info->sectorsize) - block_start; ASSERT(block_len <= U32_MAX); - /* - * Folio checked is some magic around finding folios that have been - * modified without going through btrfs_dirty_folio(). Clear it here. - * There should be no need to mark the pages accessed as - * prepare_one_folio() should have marked them accessed in - * prepare_one_folio() via find_or_create_page() - */ - btrfs_folio_clamp_clear_checked(fs_info, folio, block_start, block_len); folio_unlock(folio); folio_put(folio); } @@ -65,7 +57,7 @@ static void btrfs_drop_folio(struct btrfs_fs_info *fs_info, struct folio *folio, * After copy_folio_from_iter_atomic(), update the following things for delalloc: * - Mark newly dirtied folio as DELALLOC in the io tree. * Used to advise which range is to be written back. - * - Mark modified folio as Uptodate/Dirty and not needing COW fixup + * - Mark modified folio as Uptodate/Dirty * - Update inode size for past EOF write */ int btrfs_dirty_folio(struct btrfs_inode *inode, struct folio *folio, loff_t pos, @@ -93,21 +85,12 @@ int btrfs_dirty_folio(struct btrfs_inode *inode, struct folio *folio, loff_t pos end_of_last_block = start_pos + num_bytes - 1; - /* - * The pages may have already been dirty, clear out old accounting so - * we can set things up properly - */ - btrfs_clear_extent_bit(&inode->io_tree, start_pos, end_of_last_block, - EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, - cached); - - ret = btrfs_set_extent_delalloc(inode, start_pos, end_of_last_block, - extra_bits, cached); + ret = btrfs_reset_extent_delalloc(inode, start_pos, end_of_last_block, + extra_bits, cached); if (ret) return ret; btrfs_folio_clamp_set_uptodate(fs_info, folio, start_pos, num_bytes); - btrfs_folio_clamp_clear_checked(fs_info, folio, start_pos, num_bytes); btrfs_folio_clamp_set_dirty(fs_info, folio, start_pos, num_bytes); /* @@ -158,7 +141,7 @@ int btrfs_drop_extents(struct btrfs_trans_handle *trans, int ret; int modify_tree = -1; int update_refs; - int found = 0; + bool found = false; struct btrfs_path *path = args->path; args->bytes_found = 0; @@ -266,7 +249,7 @@ next_slot: goto next_slot; } - found = 1; + found = true; search_start = max(key.offset, args->start); if (recow || !modify_tree) { modify_tree = -1; @@ -1232,8 +1215,11 @@ static int copy_one_range(struct btrfs_inode *inode, struct iov_iter *iter, return ret; reserved_len = ret; /* Write range must be inside the reserved range. */ - ASSERT(reserved_start <= start); - ASSERT(start + write_bytes <= reserved_start + reserved_len); + ASSERT(reserved_start <= start, "reserved_start=%llu start=%llu", + reserved_start, start); + ASSERT(start + write_bytes <= reserved_start + reserved_len, + "start=%llu write_bytes=%zu reserved_start=%llu reserved_len=%llu", + start, write_bytes, reserved_start, reserved_len); again: ret = balance_dirty_pages_ratelimited_flags(inode->vfs_inode.i_mapping, @@ -1578,7 +1564,7 @@ int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync) btrfs_assert_inode_locked(inode); } - trace_btrfs_sync_file(file, datasync); + trace_btrfs_sync_file_enter(file, datasync); btrfs_init_log_ctx(&ctx, inode); @@ -1702,14 +1688,6 @@ int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync) * reason, it's no longer relevant. */ clear_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags); - /* - * An ordered extent might have started before and completed - * already with io errors, in which case the inode was not - * updated and we end up here. So check the inode's mapping - * for any errors that might have happened since we last - * checked called fsync. - */ - ret = filemap_check_wb_err(inode->vfs_inode.i_mapping, file->f_wb_err); goto out_release_extents; } @@ -1824,10 +1802,17 @@ out: free_extent_buffer(ctx.scratch_eb); ASSERT(list_empty(&ctx.list)); ASSERT(list_empty(&ctx.conflict_inodes)); + ASSERT(ret <= 0, "ret=%d", ret); + /* + * Ordered extents might have started and completed before this fsync, + * so check for any io errors and advance the writeback error sequence. + */ err = file_check_and_advance_wb_err(file); if (!ret) ret = err; - return ret > 0 ? -EIO : ret; + trace_btrfs_sync_file_exit(file, ret); + + return ret; out_release_extents: btrfs_release_log_ctx_extents(&ctx); @@ -1966,18 +1951,7 @@ again: } } - /* - * page_mkwrite gets called when the page is firstly dirtied after it's - * faulted in, but write(2) could also dirty a page and set delalloc - * bits, thus in this case for space account reason, we still need to - * clear any delalloc bits within this page range since we have to - * reserve data&meta space before lock_page() (see above comments). - */ - btrfs_clear_extent_bit(io_tree, page_start, end, - EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | - EXTENT_DEFRAG, &cached_state); - - ret = btrfs_set_extent_delalloc(inode, page_start, end, 0, &cached_state); + ret = btrfs_reset_extent_delalloc(inode, page_start, end, 0, &cached_state); if (ret < 0) { btrfs_unlock_extent(io_tree, page_start, page_end, &cached_state); goto out_unlock; @@ -1992,7 +1966,6 @@ again: if (zero_start != fsize) folio_zero_range(folio, zero_start, folio_size(folio) - zero_start); - btrfs_folio_clear_checked(fs_info, folio, page_start, fsize); btrfs_folio_set_dirty(fs_info, folio, page_start, end + 1 - page_start); btrfs_folio_set_uptodate(fs_info, folio, page_start, end + 1 - page_start); @@ -2098,6 +2071,10 @@ static int fill_holes(struct btrfs_trans_handle *trans, struct btrfs_file_extent_item *fi; struct extent_map *hole_em; struct btrfs_key key; + int modify_slot = -1; + int del_slot = -1; + bool update_offset = false; + u64 num_bytes = 0; int ret; if (btrfs_fs_incompat(fs_info, NO_HOLES)) @@ -2107,7 +2084,7 @@ static int fill_holes(struct btrfs_trans_handle *trans, key.type = BTRFS_EXTENT_DATA_KEY; key.offset = offset; - ret = btrfs_search_slot(trans, root, &key, path, 0, 1); + ret = btrfs_search_slot(trans, root, &key, path, -1, 1); if (ret <= 0) { /* * We should have dropped this offset, so if we find it then @@ -2120,33 +2097,44 @@ static int fill_holes(struct btrfs_trans_handle *trans, leaf = path->nodes[0]; if (hole_mergeable(inode, leaf, path->slots[0] - 1, offset, end)) { - u64 num_bytes; - - path->slots[0]--; - fi = btrfs_item_ptr(leaf, path->slots[0], + fi = btrfs_item_ptr(leaf, path->slots[0] - 1, struct btrfs_file_extent_item); num_bytes = btrfs_file_extent_num_bytes(leaf, fi) + end - offset; - btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes); - btrfs_set_file_extent_ram_bytes(leaf, fi, num_bytes); - btrfs_set_file_extent_offset(leaf, fi, 0); - btrfs_set_file_extent_generation(leaf, fi, trans->transid); - goto out; + modify_slot = path->slots[0] - 1; } - if (hole_mergeable(inode, leaf, path->slots[0], offset, end)) { - u64 num_bytes; - - key.offset = offset; - btrfs_set_item_key_safe(trans, path, &key); fi = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item); - num_bytes = btrfs_file_extent_num_bytes(leaf, fi) + end - - offset; + if (modify_slot != -1) { + num_bytes += btrfs_file_extent_num_bytes(leaf, fi); + del_slot = path->slots[0]; + } else { + num_bytes = btrfs_file_extent_num_bytes(leaf, fi) + + end - offset; + modify_slot = path->slots[0]; + update_offset = true; + } + } + if (modify_slot >= 0) { + fi = btrfs_item_ptr(leaf, modify_slot, + struct btrfs_file_extent_item); btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes); btrfs_set_file_extent_ram_bytes(leaf, fi, num_bytes); + if (update_offset) { + key.offset = offset; + btrfs_set_item_key_safe(trans, path, &key); + } btrfs_set_file_extent_offset(leaf, fi, 0); btrfs_set_file_extent_generation(leaf, fi, trans->transid); + if (del_slot >= 0) { + ret = btrfs_del_items(trans, root, path, del_slot, 1); + if (ret) { + btrfs_abort_transaction(trans, ret); + btrfs_release_path(path); + return ret; + } + } goto out; } btrfs_release_path(path); @@ -2407,7 +2395,7 @@ int btrfs_replace_file_extents(struct btrfs_inode *inode, struct btrfs_drop_extents_args drop_args = { 0 }; struct btrfs_root *root = inode->root; struct btrfs_fs_info *fs_info = root->fs_info; - u64 min_size = btrfs_calc_insert_metadata_size(fs_info, 1); + const u64 min_size = btrfs_calc_insert_metadata_size(fs_info, 1); u64 ino_size = round_up(inode->vfs_inode.i_size, fs_info->sectorsize); struct btrfs_trans_handle *trans = NULL; struct btrfs_block_rsv rsv; @@ -2420,7 +2408,7 @@ int btrfs_replace_file_extents(struct btrfs_inode *inode, return -EINVAL; btrfs_init_metadata_block_rsv(fs_info, &rsv, BTRFS_BLOCK_RSV_TEMP); - rsv.size = btrfs_calc_insert_metadata_size(fs_info, 1); + rsv.size = min_size; rsv.failfast = true; /* diff --git a/fs/btrfs/free-space-cache.c b/fs/btrfs/free-space-cache.c index ab22e4f9ffdd..6009b1477232 100644 --- a/fs/btrfs/free-space-cache.c +++ b/fs/btrfs/free-space-cache.c @@ -433,10 +433,6 @@ static void io_ctl_drop_pages(struct btrfs_io_ctl *io_ctl) for (i = 0; i < io_ctl->num_pages; i++) { if (io_ctl->pages[i]) { - btrfs_folio_clear_checked(io_ctl->fs_info, - page_folio(io_ctl->pages[i]), - page_offset(io_ctl->pages[i]), - PAGE_SIZE); unlock_page(io_ctl->pages[i]); put_page(io_ctl->pages[i]); } @@ -690,11 +686,12 @@ static int io_ctl_read_bitmap(struct btrfs_io_ctl *io_ctl, static void recalculate_thresholds(struct btrfs_free_space_ctl *ctl) { struct btrfs_block_group *block_group = ctl->block_group; + const int unit = block_group->fs_info->sectorsize; u64 max_bytes; u64 bitmap_bytes; u64 extent_bytes; u64 size = block_group->length; - u64 bytes_per_bg = BITS_PER_BITMAP * ctl->unit; + u64 bytes_per_bg = BITS_PER_BITMAP * unit; u64 max_bitmaps = div64_u64(size + bytes_per_bg - 1, bytes_per_bg); max_bitmaps = max_t(u64, max_bitmaps, 1); @@ -703,7 +700,7 @@ static void recalculate_thresholds(struct btrfs_free_space_ctl *ctl) btrfs_err(block_group->fs_info, "invalid free space control: bg start=%llu len=%llu total_bitmaps=%u unit=%u max_bitmaps=%llu bytes_per_bg=%llu", block_group->start, block_group->length, - ctl->total_bitmaps, ctl->unit, max_bitmaps, + ctl->total_bitmaps, unit, max_bitmaps, bytes_per_bg); ASSERT(ctl->total_bitmaps <= max_bitmaps); @@ -718,7 +715,7 @@ static void recalculate_thresholds(struct btrfs_free_space_ctl *ctl) else max_bytes = MAX_CACHE_BYTES_PER_GIG * div_u64(size, SZ_1G); - bitmap_bytes = ctl->total_bitmaps * ctl->unit; + bitmap_bytes = ctl->total_bitmaps * unit; /* * we want the extent entry threshold to always be at most 1/2 the max @@ -896,8 +893,7 @@ free_cache: goto out; } -static int copy_free_space_cache(struct btrfs_block_group *block_group, - struct btrfs_free_space_ctl *ctl) +static int copy_free_space_cache(struct btrfs_free_space_ctl *ctl) { struct btrfs_free_space *info; struct rb_node *n; @@ -912,17 +908,17 @@ static int copy_free_space_cache(struct btrfs_block_group *block_group, unlink_free_space(ctl, info, true); spin_unlock(&ctl->tree_lock); kmem_cache_free(btrfs_free_space_cachep, info); - ret = btrfs_add_free_space(block_group, offset, bytes); + ret = btrfs_add_free_space(ctl->block_group, offset, bytes); spin_lock(&ctl->tree_lock); } else { u64 offset = info->offset; - u64 bytes = ctl->unit; + u64 bytes = ctl->block_group->fs_info->sectorsize; ret = search_bitmap(ctl, info, &offset, &bytes, false); if (ret == 0) { bitmap_clear_bits(ctl, info, offset, bytes, true); spin_unlock(&ctl->tree_lock); - ret = btrfs_add_free_space(block_group, offset, + ret = btrfs_add_free_space(ctl->block_group, offset, bytes); spin_lock(&ctl->tree_lock); } else { @@ -1025,7 +1021,7 @@ int load_free_space_cache(struct btrfs_block_group *block_group) if (matched) { spin_lock(&tmp_ctl.tree_lock); - ret = copy_free_space_cache(block_group, &tmp_ctl); + ret = copy_free_space_cache(&tmp_ctl); spin_unlock(&tmp_ctl.tree_lock); /* * ret == 1 means we successfully loaded the free space cache, @@ -1068,12 +1064,12 @@ out: static noinline_for_stack int write_cache_extent_entries(struct btrfs_io_ctl *io_ctl, - struct btrfs_free_space_ctl *ctl, struct btrfs_block_group *block_group, int *entries, int *bitmaps, struct list_head *bitmap_list) { int ret; + struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; struct btrfs_free_cluster *cluster = NULL; struct btrfs_free_cluster *cluster_locked = NULL; struct rb_node *node = rb_first(&ctl->free_space_offset); @@ -1367,17 +1363,17 @@ int btrfs_wait_cache_io(struct btrfs_trans_handle *trans, * or an errno if it was not. */ static int __btrfs_write_out_cache(struct inode *inode, - struct btrfs_free_space_ctl *ctl, struct btrfs_block_group *block_group, struct btrfs_trans_handle *trans) { + struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; struct btrfs_io_ctl *io_ctl = &block_group->io_ctl; struct extent_state *cached_state = NULL; LIST_HEAD(bitmap_list); int entries = 0; int bitmaps = 0; int ret; - int must_iput = 0; + bool must_iput = false; int i_size; if (!i_size_read(inode)) @@ -1397,7 +1393,7 @@ static int __btrfs_write_out_cache(struct inode *inode, up_write(&block_group->data_rwsem); BTRFS_I(inode)->generation = 0; ret = 0; - must_iput = 1; + must_iput = true; goto out; } spin_unlock(&block_group->lock); @@ -1416,8 +1412,7 @@ static int __btrfs_write_out_cache(struct inode *inode, mutex_lock(&ctl->cache_writeout_mutex); /* Write out the extent entries in the free space cache */ spin_lock(&ctl->tree_lock); - ret = write_cache_extent_entries(io_ctl, ctl, - block_group, &entries, &bitmaps, + ret = write_cache_extent_entries(io_ctl, block_group, &entries, &bitmaps, &bitmap_list); if (ret) goto out_nospc_locked; @@ -1516,7 +1511,6 @@ int btrfs_write_out_cache(struct btrfs_trans_handle *trans, struct btrfs_path *path) { struct btrfs_fs_info *fs_info = trans->fs_info; - struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; struct inode *inode; int ret = 0; @@ -1531,7 +1525,7 @@ int btrfs_write_out_cache(struct btrfs_trans_handle *trans, if (IS_ERR(inode)) return 0; - ret = __btrfs_write_out_cache(inode, ctl, block_group, trans); + ret = __btrfs_write_out_cache(inode, block_group, trans); if (ret) { btrfs_debug(fs_info, "failed to write free space cache for block group %llu error %d", @@ -1571,11 +1565,11 @@ static inline u64 offset_to_bitmap(struct btrfs_free_space_ctl *ctl, u64 bitmap_start; u64 bytes_per_bitmap; - bytes_per_bitmap = BITS_PER_BITMAP * ctl->unit; - bitmap_start = offset - ctl->start; + bytes_per_bitmap = BITS_PER_BITMAP * ctl->block_group->fs_info->sectorsize; + bitmap_start = offset - ctl->block_group->start; bitmap_start = div64_u64(bitmap_start, bytes_per_bitmap); bitmap_start *= bytes_per_bitmap; - bitmap_start += ctl->start; + bitmap_start += ctl->block_group->start; return bitmap_start; } @@ -1702,6 +1696,7 @@ tree_search_offset(struct btrfs_free_space_ctl *ctl, { struct rb_node *n = ctl->free_space_offset.rb_node; struct btrfs_free_space *entry = NULL, *prev = NULL; + const int unit = ctl->block_group->fs_info->sectorsize; lockdep_assert_held(&ctl->tree_lock); @@ -1785,7 +1780,7 @@ tree_search_offset(struct btrfs_free_space_ctl *ctl, prev->offset + prev->bytes > offset) return prev; } - if (entry->offset + BITS_PER_BITMAP * ctl->unit > offset) + if (entry->offset + BITS_PER_BITMAP * unit > offset) return entry; } else if (entry->offset + entry->bytes > offset) return entry; @@ -1799,8 +1794,7 @@ tree_search_offset(struct btrfs_free_space_ctl *ctl, return NULL; entry = rb_entry(n, struct btrfs_free_space, offset_index); if (entry->bitmap) { - if (entry->offset + BITS_PER_BITMAP * - ctl->unit > offset) + if (entry->offset + BITS_PER_BITMAP * unit > offset) break; } else { if (entry->offset + entry->bytes > offset) @@ -1875,18 +1869,19 @@ static inline void bitmap_clear_bits(struct btrfs_free_space_ctl *ctl, struct btrfs_free_space *info, u64 offset, u64 bytes, bool update_stat) { + const int unit = ctl->block_group->fs_info->sectorsize; unsigned long start, count, end; int extent_delta = -1; - start = offset_to_bit(info->offset, ctl->unit, offset); - count = bytes_to_bits(bytes, ctl->unit); + start = offset_to_bit(info->offset, unit, offset); + count = bytes_to_bits(bytes, unit); end = start + count; ASSERT(end <= BITS_PER_BITMAP); bitmap_clear(info->bitmap, start, count); info->bytes -= bytes; - if (info->max_extent_size > ctl->unit) + if (info->max_extent_size > unit) info->max_extent_size = 0; relink_bitmap_entry(ctl, info); @@ -1911,11 +1906,12 @@ static void btrfs_bitmap_set_bits(struct btrfs_free_space_ctl *ctl, struct btrfs_free_space *info, u64 offset, u64 bytes) { + const int unit = ctl->block_group->fs_info->sectorsize; unsigned long start, count, end; int extent_delta = 1; - start = offset_to_bit(info->offset, ctl->unit, offset); - count = bytes_to_bits(bytes, ctl->unit); + start = offset_to_bit(info->offset, unit, offset); + count = bytes_to_bits(bytes, unit); end = start + count; ASSERT(end <= BITS_PER_BITMAP); @@ -1952,6 +1948,7 @@ static int search_bitmap(struct btrfs_free_space_ctl *ctl, struct btrfs_free_space *bitmap_info, u64 *offset, u64 *bytes, bool for_alloc) { + const int unit = ctl->block_group->fs_info->sectorsize; unsigned long found_bits = 0; unsigned long max_bits = 0; unsigned long bits, i; @@ -1969,9 +1966,9 @@ static int search_bitmap(struct btrfs_free_space_ctl *ctl, return -1; } - i = offset_to_bit(bitmap_info->offset, ctl->unit, + i = offset_to_bit(bitmap_info->offset, unit, max_t(u64, *offset, bitmap_info->offset)); - bits = bytes_to_bits(*bytes, ctl->unit); + bits = bytes_to_bits(*bytes, unit); for_each_set_bit_from(i, bitmap_info->bitmap, BITS_PER_BITMAP) { if (for_alloc && bits == 1) { @@ -1991,12 +1988,12 @@ static int search_bitmap(struct btrfs_free_space_ctl *ctl, } if (found_bits) { - *offset = (u64)(i * ctl->unit) + bitmap_info->offset; - *bytes = (u64)(found_bits) * ctl->unit; + *offset = (u64)(i * unit) + bitmap_info->offset; + *bytes = (u64)(found_bits) * unit; return 0; } - *bytes = (u64)(max_bits) * ctl->unit; + *bytes = (u64)(max_bits) * unit; bitmap_info->max_extent_size = *bytes; relink_bitmap_entry(ctl, bitmap_info); return -1; @@ -2054,9 +2051,9 @@ again: * to match our requested alignment */ if (*bytes >= align) { - tmp = entry->offset - ctl->start + align - 1; + tmp = entry->offset - ctl->block_group->start + align - 1; tmp = div64_u64(tmp, align); - tmp = tmp * align + ctl->start; + tmp = tmp * align + ctl->block_group->start; align_off = tmp - entry->offset; } else { align_off = 0; @@ -2148,12 +2145,13 @@ static noinline int remove_from_bitmap(struct btrfs_free_space_ctl *ctl, struct btrfs_free_space *bitmap_info, u64 *offset, u64 *bytes) { + const int unit = ctl->block_group->fs_info->sectorsize; u64 end; u64 search_start, search_bytes; int ret; again: - end = bitmap_info->offset + (u64)(BITS_PER_BITMAP * ctl->unit) - 1; + end = bitmap_info->offset + (u64)(BITS_PER_BITMAP * unit) - 1; /* * We need to search for bits in this bitmap. We could only cover some @@ -2162,7 +2160,7 @@ again: * go searching for the next bit. */ search_start = *offset; - search_bytes = ctl->unit; + search_bytes = unit; search_bytes = min(search_bytes, end - search_start + 1); ret = search_bitmap(ctl, bitmap_info, &search_start, &search_bytes, false); @@ -2208,7 +2206,7 @@ again: * everything over again. */ search_start = *offset; - search_bytes = ctl->unit; + search_bytes = unit; ret = search_bitmap(ctl, bitmap_info, &search_start, &search_bytes, false); if (ret < 0 || search_start != *offset) @@ -2225,6 +2223,7 @@ static u64 add_bytes_to_bitmap(struct btrfs_free_space_ctl *ctl, struct btrfs_free_space *info, u64 offset, u64 bytes, enum btrfs_trim_state trim_state) { + const int unit = ctl->block_group->fs_info->sectorsize; u64 bytes_to_set = 0; u64 end; @@ -2241,7 +2240,7 @@ static u64 add_bytes_to_bitmap(struct btrfs_free_space_ctl *ctl, info->trim_state = BTRFS_TRIM_STATE_UNTRIMMED; } - end = info->offset + (u64)(BITS_PER_BITMAP * ctl->unit); + end = info->offset + (u64)(BITS_PER_BITMAP * unit); bytes_to_set = min(end - offset, bytes); @@ -2251,7 +2250,8 @@ static u64 add_bytes_to_bitmap(struct btrfs_free_space_ctl *ctl, } -static bool use_bitmap(struct btrfs_free_space_ctl *ctl, +EXPORT_FOR_TESTS +bool btrfs_use_bitmap(struct btrfs_free_space_ctl *ctl, struct btrfs_free_space *info) { struct btrfs_block_group *block_group = ctl->block_group; @@ -2295,22 +2295,18 @@ static bool use_bitmap(struct btrfs_free_space_ctl *ctl, * so allow those block groups to still be allowed to have a bitmap * entry. */ - if (((BITS_PER_BITMAP * ctl->unit) >> 1) > block_group->length) + if (((BITS_PER_BITMAP * fs_info->sectorsize) >> 1) > block_group->length) return false; return true; } -static const struct btrfs_free_space_op free_space_op = { - .use_bitmap = use_bitmap, -}; - static int insert_into_bitmap(struct btrfs_free_space_ctl *ctl, struct btrfs_free_space *info) { struct btrfs_free_space *bitmap_info; - struct btrfs_block_group *block_group = NULL; - int added = 0; + struct btrfs_block_group *block_group = ctl->block_group; + bool added = false; u64 bytes, offset, bytes_added; enum btrfs_trim_state trim_state; int ret; @@ -2319,18 +2315,20 @@ static int insert_into_bitmap(struct btrfs_free_space_ctl *ctl, offset = info->offset; trim_state = info->trim_state; - if (!ctl->op->use_bitmap(ctl, info)) - return 0; - - if (ctl->op == &free_space_op) - block_group = ctl->block_group; + if (btrfs_is_testing(block_group->fs_info)) { + if (!block_group->fs_info->use_bitmap(ctl, info)) + return 0; + } else { + if (!btrfs_use_bitmap(ctl, info)) + return 0; + } again: /* * Since we link bitmaps right into the cluster we need to see if we * have a cluster here, and if so and it has our bitmap we need to add * the free space to that bitmap. */ - if (block_group && !list_empty(&block_group->cluster_list)) { + if (!list_empty(&block_group->cluster_list)) { struct btrfs_free_cluster *cluster; struct rb_node *node; struct btrfs_free_space *entry; @@ -2367,7 +2365,7 @@ no_cluster_bitmap: bitmap_info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset), 1, 0); if (!bitmap_info) { - ASSERT(added == 0); + ASSERT(!added); goto new_bitmap; } @@ -2375,7 +2373,7 @@ no_cluster_bitmap: trim_state); bytes -= bytes_added; offset += bytes_added; - added = 0; + added = false; if (!bytes) { ret = 1; @@ -2386,7 +2384,7 @@ no_cluster_bitmap: new_bitmap: if (info && info->bitmap) { add_new_bitmap(ctl, info, offset); - added = 1; + added = true; info = NULL; goto again; } else { @@ -2494,6 +2492,7 @@ static bool steal_from_bitmap_to_end(struct btrfs_free_space_ctl *ctl, struct btrfs_free_space *info, bool update_stat) { + const int unit = ctl->block_group->fs_info->sectorsize; struct btrfs_free_space *bitmap; unsigned long i; unsigned long j; @@ -2505,11 +2504,11 @@ static bool steal_from_bitmap_to_end(struct btrfs_free_space_ctl *ctl, if (!bitmap) return false; - i = offset_to_bit(bitmap->offset, ctl->unit, end); + i = offset_to_bit(bitmap->offset, unit, end); j = find_next_zero_bit(bitmap->bitmap, BITS_PER_BITMAP, i); if (j == i) return false; - bytes = (j - i) * ctl->unit; + bytes = (j - i) * unit; info->bytes += bytes; /* See try_merge_free_space() comment. */ @@ -2528,6 +2527,7 @@ static bool steal_from_bitmap_to_front(struct btrfs_free_space_ctl *ctl, struct btrfs_free_space *info, bool update_stat) { + const int unit = ctl->block_group->fs_info->sectorsize; struct btrfs_free_space *bitmap; u64 bitmap_offset; unsigned long i; @@ -2547,7 +2547,7 @@ static bool steal_from_bitmap_to_front(struct btrfs_free_space_ctl *ctl, if (!bitmap) return false; - i = offset_to_bit(bitmap->offset, ctl->unit, info->offset) - 1; + i = offset_to_bit(bitmap->offset, unit, info->offset) - 1; j = 0; prev_j = (unsigned long)-1; for_each_clear_bit_from(j, bitmap->bitmap, BITS_PER_BITMAP) { @@ -2559,9 +2559,9 @@ static bool steal_from_bitmap_to_front(struct btrfs_free_space_ctl *ctl, return false; if (prev_j == (unsigned long)-1) - bytes = (i + 1) * ctl->unit; + bytes = (i + 1) * unit; else - bytes = (i - prev_j) * ctl->unit; + bytes = (i - prev_j) * unit; info->offset -= bytes; info->bytes += bytes; @@ -2947,13 +2947,8 @@ void btrfs_dump_free_space(struct btrfs_block_group *block_group, void btrfs_init_free_space_ctl(struct btrfs_block_group *block_group, struct btrfs_free_space_ctl *ctl) { - struct btrfs_fs_info *fs_info = block_group->fs_info; - spin_lock_init(&ctl->tree_lock); - ctl->unit = fs_info->sectorsize; - ctl->start = block_group->start; ctl->block_group = block_group; - ctl->op = &free_space_op; ctl->free_space_bytes = RB_ROOT_CACHED; INIT_LIST_HEAD(&ctl->trimming_ranges); mutex_init(&ctl->cache_writeout_mutex); @@ -3327,6 +3322,7 @@ static int btrfs_bitmap_cluster(struct btrfs_block_group *block_group, u64 cont1_bytes, u64 min_bytes) { struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; + const int unit = block_group->fs_info->sectorsize; unsigned long next_zero; unsigned long i; unsigned long want_bits; @@ -3339,10 +3335,10 @@ static int btrfs_bitmap_cluster(struct btrfs_block_group *block_group, lockdep_assert_held(&ctl->tree_lock); - i = offset_to_bit(entry->offset, ctl->unit, + i = offset_to_bit(entry->offset, unit, max_t(u64, offset, entry->offset)); - want_bits = bytes_to_bits(bytes, ctl->unit); - min_bits = bytes_to_bits(min_bytes, ctl->unit); + want_bits = bytes_to_bits(bytes, unit); + min_bits = bytes_to_bits(min_bytes, unit); /* * Don't bother looking for a cluster in this bitmap if it's heavily @@ -3368,7 +3364,7 @@ again: } if (!found_bits) { - entry->max_extent_size = (u64)max_bits * ctl->unit; + entry->max_extent_size = (u64)max_bits * unit; return -ENOSPC; } @@ -3379,15 +3375,15 @@ again: total_found += found_bits; - if (cluster->max_size < found_bits * ctl->unit) - cluster->max_size = found_bits * ctl->unit; + if (cluster->max_size < found_bits * unit) + cluster->max_size = found_bits * unit; if (total_found < want_bits || cluster->max_size < cont1_bytes) { i = next_zero + 1; goto again; } - cluster->window_start = start * ctl->unit + entry->offset; + cluster->window_start = start * unit + entry->offset; rb_erase(&entry->offset_index, &ctl->free_space_offset); rb_erase_cached(&entry->bytes_index, &ctl->free_space_bytes); @@ -3403,8 +3399,7 @@ again: ret = tree_insert_offset(ctl, cluster, entry); ASSERT(!ret); /* -EEXIST; Logic error */ - trace_btrfs_setup_cluster(block_group, cluster, - total_found * ctl->unit, 1); + trace_btrfs_setup_cluster(block_group, cluster, total_found * unit, 1); return 0; } @@ -4044,7 +4039,9 @@ static int trim_bitmaps(struct btrfs_block_group *block_group, } next: if (next_bitmap) { - offset += BITS_PER_BITMAP * ctl->unit; + const int unit = block_group->fs_info->sectorsize; + + offset += BITS_PER_BITMAP * unit; start = offset; } else { start += bytes; @@ -4071,6 +4068,7 @@ int btrfs_trim_block_group(struct btrfs_block_group *block_group, u64 *trimmed, u64 start, u64 end, u64 minlen) { struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; + const int unit = block_group->fs_info->sectorsize; int ret; u64 rem = 0; @@ -4091,7 +4089,7 @@ int btrfs_trim_block_group(struct btrfs_block_group *block_group, goto out; ret = trim_bitmaps(block_group, trimmed, start, end, minlen, 0, false); - div64_u64_rem(end, BITS_PER_BITMAP * ctl->unit, &rem); + div64_u64_rem(end, BITS_PER_BITMAP * unit, &rem); /* If we ended in the middle of a bitmap, reset the trimming flag */ if (rem) reset_trimming_bitmap(ctl, offset_to_bitmap(ctl, end)); @@ -4310,6 +4308,7 @@ int test_check_exists(struct btrfs_block_group *cache, u64 offset, u64 bytes) { struct btrfs_free_space_ctl *ctl = cache->free_space_ctl; + const int unit = cache->fs_info->sectorsize; struct btrfs_free_space *info; int ret = 0; @@ -4329,7 +4328,7 @@ have_info: struct btrfs_free_space *tmp; bit_off = offset; - bit_bytes = ctl->unit; + bit_bytes = unit; ret = search_bitmap(ctl, info, &bit_off, &bit_bytes, false); if (!ret) { if (bit_off == offset) { diff --git a/fs/btrfs/free-space-cache.h b/fs/btrfs/free-space-cache.h index 33fc3b245648..53fe8e293af1 100644 --- a/fs/btrfs/free-space-cache.h +++ b/fs/btrfs/free-space-cache.h @@ -74,28 +74,20 @@ enum { }; struct btrfs_free_space_ctl { - spinlock_t tree_lock; struct rb_root free_space_offset; struct rb_root_cached free_space_bytes; - u64 free_space; + spinlock_t tree_lock; int extents_thresh; int free_extents; int total_bitmaps; - int unit; - u64 start; + u64 free_space; s32 discardable_extents[BTRFS_STAT_NR_ENTRIES]; s64 discardable_bytes[BTRFS_STAT_NR_ENTRIES]; - const struct btrfs_free_space_op *op; struct btrfs_block_group *block_group; struct mutex cache_writeout_mutex; struct list_head trimming_ranges; }; -struct btrfs_free_space_op { - bool (*use_bitmap)(struct btrfs_free_space_ctl *ctl, - struct btrfs_free_space *info); -}; - struct btrfs_io_ctl { void *cur, *orig; struct page *page; @@ -172,6 +164,8 @@ bool btrfs_free_space_cache_v1_active(struct btrfs_fs_info *fs_info); int btrfs_set_free_space_cache_v1_active(struct btrfs_fs_info *fs_info, bool active); /* Support functions for running our sanity tests */ #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS +bool btrfs_use_bitmap(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *info); int test_add_free_space_entry(struct btrfs_block_group *cache, u64 offset, u64 bytes, bool bitmap); int test_check_exists(struct btrfs_block_group *cache, u64 offset, u64 bytes); diff --git a/fs/btrfs/free-space-tree.c b/fs/btrfs/free-space-tree.c index 472b3060e5ac..1b3d82ae3de8 100644 --- a/fs/btrfs/free-space-tree.c +++ b/fs/btrfs/free-space-tree.c @@ -109,7 +109,7 @@ struct btrfs_free_space_info *btrfs_search_free_space_info( ret = btrfs_search_slot(trans, root, &key, path, 0, cow); if (ret < 0) return ERR_PTR(ret); - if (ret != 0) { + if (unlikely(ret != 0)) { btrfs_warn(fs_info, "missing free space info for %llu", block_group->start); DEBUG_WARN(); @@ -209,7 +209,8 @@ int btrfs_convert_free_space_to_bitmaps(struct btrfs_trans_handle *trans, u64 bitmap_range, i; u32 bitmap_size, flags, expected_extent_count; u32 extent_count = 0; - int done = 0, nr; + bool done = false; + int nr; int ret; bitmap_size = free_space_bitmap_size(fs_info, block_group->length); @@ -240,7 +241,7 @@ int btrfs_convert_free_space_to_bitmaps(struct btrfs_trans_handle *trans, if (found_key.type == BTRFS_FREE_SPACE_INFO_KEY) { ASSERT(found_key.objectid == block_group->start); ASSERT(found_key.offset == block_group->length); - done = 1; + done = true; break; } else if (found_key.type == BTRFS_FREE_SPACE_EXTENT_KEY) { u64 first, last; @@ -353,7 +354,8 @@ int btrfs_convert_free_space_to_extents(struct btrfs_trans_handle *trans, u32 bitmap_size, flags, expected_extent_count; unsigned long nrbits, start_bit, end_bit; u32 extent_count = 0; - int done = 0, nr; + bool done = false; + int nr; int ret; bitmap_size = free_space_bitmap_size(fs_info, block_group->length); @@ -384,7 +386,7 @@ int btrfs_convert_free_space_to_extents(struct btrfs_trans_handle *trans, if (found_key.type == BTRFS_FREE_SPACE_INFO_KEY) { ASSERT(found_key.objectid == block_group->start); ASSERT(found_key.offset == block_group->length); - done = 1; + done = true; break; } else if (found_key.type == BTRFS_FREE_SPACE_BITMAP_KEY) { unsigned long ptr; @@ -1473,7 +1475,8 @@ int btrfs_remove_block_group_free_space(struct btrfs_trans_handle *trans, struct btrfs_key key, found_key; struct extent_buffer *leaf; u64 start, end; - int done = 0, nr; + bool done = false; + int nr; int ret; if (!btrfs_fs_compat_ro(trans->fs_info, FREE_SPACE_TREE)) @@ -1514,7 +1517,7 @@ int btrfs_remove_block_group_free_space(struct btrfs_trans_handle *trans, if (found_key.type == BTRFS_FREE_SPACE_INFO_KEY) { ASSERT(found_key.objectid == block_group->start); ASSERT(found_key.offset == block_group->length); - done = 1; + done = true; nr++; path->slots[0]--; break; @@ -1545,6 +1548,29 @@ int btrfs_remove_block_group_free_space(struct btrfs_trans_handle *trans, return 0; } +static int validate_free_space_key(struct btrfs_block_group *block_group, + const struct btrfs_key *key, u8 expected_type) +{ + const u64 end = btrfs_block_group_end(block_group); + + if (unlikely(key->type != expected_type)) { + btrfs_err(block_group->fs_info, + "block group %llu has unexpected free space key type %u, expected %u", + block_group->start, key->type, expected_type); + return -EUCLEAN; + } + + if (unlikely(key->objectid + key->offset > end)) { + btrfs_err(block_group->fs_info, + "block group %llu has invalid free space key (%llu %u %llu)", + block_group->start, key->objectid, key->type, + key->offset); + return -EUCLEAN; + } + + return 0; +} + static int load_free_space_bitmaps(struct btrfs_caching_control *caching_ctl, struct btrfs_path *path, u32 expected_extent_count) @@ -1576,8 +1602,9 @@ static int load_free_space_bitmaps(struct btrfs_caching_control *caching_ctl, if (key.type == BTRFS_FREE_SPACE_INFO_KEY) break; - ASSERT(key.type == BTRFS_FREE_SPACE_BITMAP_KEY); - ASSERT(key.objectid < end && key.objectid + key.offset <= end); + ret = validate_free_space_key(block_group, &key, BTRFS_FREE_SPACE_BITMAP_KEY); + if (unlikely(ret)) + return ret; offset = key.objectid; while (offset < key.objectid + key.offset) { @@ -1633,7 +1660,6 @@ static int load_free_space_extents(struct btrfs_caching_control *caching_ctl, struct btrfs_fs_info *fs_info = block_group->fs_info; struct btrfs_root *root; struct btrfs_key key; - const u64 end = btrfs_block_group_end(block_group); u64 total_found = 0; u32 extent_count = 0; int ret; @@ -1654,8 +1680,9 @@ static int load_free_space_extents(struct btrfs_caching_control *caching_ctl, if (key.type == BTRFS_FREE_SPACE_INFO_KEY) break; - ASSERT(key.type == BTRFS_FREE_SPACE_EXTENT_KEY); - ASSERT(key.objectid < end && key.objectid + key.offset <= end); + ret = validate_free_space_key(block_group, &key, BTRFS_FREE_SPACE_EXTENT_KEY); + if (unlikely(ret)) + return ret; ret = btrfs_add_new_free_space(block_group, key.objectid, key.objectid + key.offset, diff --git a/fs/btrfs/fs.h b/fs/btrfs/fs.h index a8aa086a4df8..5f0cfb0b5466 100644 --- a/fs/btrfs/fs.h +++ b/fs/btrfs/fs.h @@ -50,20 +50,26 @@ struct btrfs_subpage_info; struct btrfs_stripe_hash_table; struct btrfs_space_info; -/* - * Minimum data and metadata block size. - * - * Normally it's 4K, but for testing subpage block size on 4K page systems, we - * allow DEBUG builds to accept 2K page size. - */ -#ifdef CONFIG_BTRFS_DEBUG -#define BTRFS_MIN_BLOCKSIZE (SZ_2K) -#else +/* Minimum data and metadata block size. */ #define BTRFS_MIN_BLOCKSIZE (SZ_4K) -#endif - #define BTRFS_MAX_BLOCKSIZE (SZ_64K) +/* The maximum folio size btrfs supports. */ +#define BTRFS_MAX_FOLIO_SIZE (SZ_2M) +static_assert(BTRFS_MAX_FOLIO_SIZE > PAGE_SIZE); + +/* + * The maximum number of blocks a huge folio can support. + * + * Depending on the filesystem block size, the real maximum blocks per folio + * may also be limited by the above BTRFS_MAX_FOLIO_SIZE. + */ +#ifdef CONFIG_BTRFS_EXPERIMENTAL +#define BTRFS_MAX_BLOCKS_PER_FOLIO (512) +#else +#define BTRFS_MAX_BLOCKS_PER_FOLIO (BITS_PER_LONG) +#endif + #define BTRFS_MAX_EXTENT_SIZE SZ_128M /* @@ -89,6 +95,10 @@ static_assert(sizeof(struct btrfs_super_block) == BTRFS_SUPER_INFO_SIZE); #define BTRFS_KEY_FMT "(%llu %u %llu)" #define BTRFS_KEY_FMT_VALUE(key) (key)->objectid, (key)->type, (key)->offset +#define BTRFS_QGROUP_FMT "%hu/%llu" +#define BTRFS_QGROUP_FMT_VALUE(qgroup) btrfs_qgroup_level((qgroup)->qgroupid), \ + btrfs_qgroup_subvolid((qgroup)->qgroupid) + /* * Number of metadata items necessary for an unlink operation: * @@ -486,6 +496,9 @@ struct btrfs_delayed_root { wait_queue_head_t wait; }; +struct btrfs_free_space_ctl; +struct btrfs_free_space; + struct btrfs_fs_info { u8 chunk_tree_uuid[BTRFS_UUID_SIZE]; unsigned long flags; @@ -644,6 +657,8 @@ struct btrfs_fs_info { * to protect us from the relocation code. */ struct mutex reloc_mutex; + /* Protects setting, clearing and getting fs_info->reloc_ctl. */ + spinlock_t reloc_ctl_lock; struct list_head trans_list; struct list_head dead_roots; @@ -698,13 +713,6 @@ struct btrfs_fs_info { struct btrfs_workqueue *endio_write_workers; struct btrfs_workqueue *endio_freespace_worker; struct btrfs_workqueue *caching_workers; - - /* - * Fixup workers take dirty pages that didn't properly go through the - * cow mechanism and make them safe to write. It happens for the - * sys_munmap function call path. - */ - struct btrfs_workqueue *fixup_workers; struct btrfs_workqueue *delayed_workers; struct task_struct *transaction_kthread; @@ -881,6 +889,7 @@ struct btrfs_fs_info { u32 block_min_order; u32 block_max_order; u32 stripesize; + u32 writeback_bio_size; u32 csum_size; u32 csums_per_leaf; u32 csum_type; @@ -958,6 +967,10 @@ struct btrfs_fs_info { spinlock_t eb_leak_lock; struct list_head allocated_ebs; #endif + + /* Used by self tests only. */ + bool (*use_bitmap)(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *info); }; #define folio_to_inode(_folio) (BTRFS_I(_Generic((_folio), \ @@ -1207,14 +1220,6 @@ static inline void btrfs_force_shutdown(struct btrfs_fs_info *fs_info) } } -/* - * We use folio flag owner_2 to indicate there is an ordered extent with - * unfinished IO. - */ -#define folio_test_ordered(folio) folio_test_owner_2(folio) -#define folio_set_ordered(folio) folio_set_owner_2(folio) -#define folio_clear_ordered(folio) folio_clear_owner_2(folio) - #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS #define EXPORT_FOR_TESTS diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 509042adee07..272598f6ae77 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -226,9 +226,7 @@ static void print_data_reloc_error(const struct btrfs_inode *inode, u64 file_off u32 item_size; int ret; - mutex_lock(&fs_info->reloc_mutex); logical = btrfs_get_reloc_bg_bytenr(fs_info); - mutex_unlock(&fs_info->reloc_mutex); if (logical == U64_MAX) { btrfs_warn_rl(fs_info, "has data reloc tree but no running relocation"); @@ -401,28 +399,6 @@ void btrfs_inode_unlock(struct btrfs_inode *inode, unsigned int ilock_flags) static inline void btrfs_cleanup_ordered_extents(struct btrfs_inode *inode, u64 offset, u64 bytes) { - pgoff_t index = offset >> PAGE_SHIFT; - const pgoff_t end_index = (offset + bytes - 1) >> PAGE_SHIFT; - struct folio *folio; - - while (index <= end_index) { - folio = filemap_get_folio(inode->vfs_inode.i_mapping, index); - if (IS_ERR(folio)) { - index++; - continue; - } - - index = folio_next_index(folio); - /* - * Here we just clear all Ordered bits for every page in the - * range, then btrfs_mark_ordered_io_finished() will handle - * the ordered extent accounting for the range. - */ - btrfs_folio_clamp_clear_ordered(inode->root->fs_info, folio, - offset, bytes); - folio_put(folio); - } - return btrfs_mark_ordered_io_finished(inode, offset, bytes, false); } @@ -755,7 +731,7 @@ static inline int inode_need_compress(struct btrfs_inode *inode, u64 start, { struct btrfs_fs_info *fs_info = inode->root->fs_info; - if (!btrfs_inode_can_compress(inode)) { + if (unlikely(!btrfs_inode_can_compress(inode))) { DEBUG_WARN("BTRFS: unexpected compression for ino %llu", btrfs_ino(inode)); return 0; } @@ -842,7 +818,7 @@ static struct folio *compressed_bio_last_folio(struct compressed_bio *cb) ASSERT(bio->bi_vcnt); bvec = &bio->bi_io_vec[bio->bi_vcnt - 1]; - paddr = page_to_phys(bvec->bv_page) + bvec->bv_offset + bvec->bv_len - 1; + paddr = bvec_phys(bvec) + bvec->bv_len - 1; return page_folio(phys_to_page(paddr)); } @@ -1406,7 +1382,6 @@ static noinline int cow_file_range(struct btrfs_inode *inode, * setup for writepage. */ page_ops = ((flags & COW_FILE_RANGE_KEEP_LOCKED) ? 0 : PAGE_UNLOCK); - page_ops |= PAGE_SET_ORDERED; /* * Relocation relies on the relocated extents to have exactly the same @@ -1972,8 +1947,7 @@ static int nocow_one_range(struct btrfs_inode *inode, struct folio *locked_folio goto error; extent_clear_unlock_delalloc(inode, file_pos, end, locked_folio, cached, EXTENT_LOCKED | EXTENT_DELALLOC | - EXTENT_CLEAR_DATA_RESV, - PAGE_SET_ORDERED); + EXTENT_CLEAR_DATA_RESV, 0); return ret; error: @@ -2317,7 +2291,7 @@ error: static bool should_nocow(struct btrfs_inode *inode, u64 start, u64 end) { if (inode->flags & (BTRFS_INODE_NODATACOW | BTRFS_INODE_PREALLOC)) { - if (inode->defrag_bytes && + if (data_race(inode->defrag_bytes) && btrfs_test_range_bit_exists(&inode->io_tree, start, end, EXTENT_DEFRAG)) return false; return true; @@ -2605,8 +2579,7 @@ void btrfs_set_delalloc_extent(struct btrfs_inode *inode, struct extent_state *s lockdep_assert_held(&inode->io_tree.lock); - if ((bits & EXTENT_DEFRAG) && !(bits & EXTENT_DELALLOC)) - WARN_ON(1); + WARN_ON((bits & EXTENT_DEFRAG) && !(bits & EXTENT_DELALLOC)); /* * set_bit and clear bit hooks normally require _irqsave/restore * but in this case, we are only testing for the DELALLOC @@ -2810,7 +2783,13 @@ int btrfs_set_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end, unsigned int extra_bits, struct extent_state **cached_state) { - WARN_ON(PAGE_ALIGNED(end)); + const u32 blocksize = inode->root->fs_info->sectorsize; + + /* Basic alignment check. */ + ASSERT(IS_ALIGNED(start, blocksize), "start=%llu blocksize=%u", + start, blocksize); + ASSERT(IS_ALIGNED(end + 1, blocksize), "inclusive end=%llu blocksize=%u", + end, blocksize); if (start >= i_size_read(&inode->vfs_inode) && !(inode->flags & BTRFS_INODE_PREALLOC)) { @@ -2833,206 +2812,50 @@ int btrfs_set_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end, EXTENT_DELALLOC | extra_bits, cached_state); } -/* see btrfs_writepage_start_hook for details on why this is required */ -struct btrfs_writepage_fixup { - struct folio *folio; - struct btrfs_inode *inode; - struct btrfs_work work; -}; - -static void btrfs_writepage_fixup_worker(struct btrfs_work *work) -{ - struct btrfs_writepage_fixup *fixup = - container_of(work, struct btrfs_writepage_fixup, work); - struct btrfs_ordered_extent *ordered; - struct extent_state *cached_state = NULL; - struct extent_changeset *data_reserved = NULL; - struct folio *folio = fixup->folio; - struct btrfs_inode *inode = fixup->inode; - struct btrfs_fs_info *fs_info = inode->root->fs_info; - u64 page_start = folio_pos(folio); - u64 page_end = folio_next_pos(folio) - 1; - int ret = 0; - bool free_delalloc_space = true; - - /* - * This is similar to page_mkwrite, we need to reserve the space before - * we take the folio lock. - */ - ret = btrfs_delalloc_reserve_space(inode, &data_reserved, page_start, - folio_size(folio)); -again: - folio_lock(folio); - - /* - * Before we queued this fixup, we took a reference on the folio. - * folio->mapping may go NULL, but it shouldn't be moved to a different - * address space. - */ - if (!folio->mapping || !folio_test_dirty(folio) || - !folio_test_checked(folio)) { - /* - * Unfortunately this is a little tricky, either - * - * 1) We got here and our folio had already been dealt with and - * we reserved our space, thus ret == 0, so we need to just - * drop our space reservation and bail. This can happen the - * first time we come into the fixup worker, or could happen - * while waiting for the ordered extent. - * 2) Our folio was already dealt with, but we happened to get an - * ENOSPC above from the btrfs_delalloc_reserve_space. In - * this case we obviously don't have anything to release, but - * because the folio was already dealt with we don't want to - * mark the folio with an error, so make sure we're resetting - * ret to 0. This is why we have this check _before_ the ret - * check, because we do not want to have a surprise ENOSPC - * when the folio was already properly dealt with. - */ - if (!ret) { - btrfs_delalloc_release_extents(inode, folio_size(folio)); - btrfs_delalloc_release_space(inode, data_reserved, - page_start, folio_size(folio), - true); - } - ret = 0; - goto out_page; - } - - /* - * We can't mess with the folio state unless it is locked, so now that - * it is locked bail if we failed to make our space reservation. - */ - if (ret) - goto out_page; - - btrfs_lock_extent(&inode->io_tree, page_start, page_end, &cached_state); - - /* already ordered? We're done */ - if (folio_test_ordered(folio)) - goto out_reserved; - - ordered = btrfs_lookup_ordered_range(inode, page_start, PAGE_SIZE); - if (ordered) { - btrfs_unlock_extent(&inode->io_tree, page_start, page_end, - &cached_state); - folio_unlock(folio); - btrfs_start_ordered_extent(ordered); - btrfs_put_ordered_extent(ordered); - goto again; - } - - ret = btrfs_set_extent_delalloc(inode, page_start, page_end, 0, - &cached_state); - if (ret) - goto out_reserved; - - /* - * Everything went as planned, we're now the owner of a dirty page with - * delayed allocation bits set and space reserved for our COW - * destination. - * - * The page was dirty when we started, nothing should have cleaned it. - */ - BUG_ON(!folio_test_dirty(folio)); - free_delalloc_space = false; -out_reserved: - btrfs_delalloc_release_extents(inode, PAGE_SIZE); - if (free_delalloc_space) - btrfs_delalloc_release_space(inode, data_reserved, page_start, - PAGE_SIZE, true); - btrfs_unlock_extent(&inode->io_tree, page_start, page_end, &cached_state); -out_page: - if (ret) { - /* - * We hit ENOSPC or other errors. Update the mapping and page - * to reflect the errors and clean the page. - */ - mapping_set_error(folio->mapping, ret); - btrfs_folio_clear_ordered(fs_info, folio, page_start, - folio_size(folio)); - btrfs_mark_ordered_io_finished(inode, page_start, - folio_size(folio), !ret); - folio_clear_dirty_for_io(folio); - } - btrfs_folio_clear_checked(fs_info, folio, page_start, PAGE_SIZE); - folio_unlock(folio); - folio_put(folio); - kfree(fixup); - extent_changeset_free(data_reserved); - /* - * As a precaution, do a delayed iput in case it would be the last iput - * that could need flushing space. Recursing back to fixup worker would - * deadlock. - */ - btrfs_add_delayed_iput(inode); -} - /* - * There are a few paths in the higher layers of the kernel that directly - * set the folio dirty bit without asking the filesystem if it is a - * good idea. This causes problems because we want to make sure COW - * properly happens and the data=ordered rules are followed. + * Clear the old accounting flags and set EXTENT_DELALLOC for the range. * - * In our case any range that doesn't have the ORDERED bit set - * hasn't been properly setup for IO. We kick off an async process - * to fix it up. The async helper will wait for ordered extents, set - * the delalloc bit and make it safe to write the folio. + * Return <0 for error, in that case no range has EXTENT_DELALLOC bit cleared or set. */ -int btrfs_writepage_cow_fixup(struct folio *folio) +int btrfs_reset_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end, + unsigned int extra_bits, struct extent_state **cached_state) { - struct inode *inode = folio->mapping->host; - struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); - struct btrfs_writepage_fixup *fixup; + const u32 blocksize = inode->root->fs_info->sectorsize; - /* This folio has ordered extent covering it already */ - if (folio_test_ordered(folio)) - return 0; + /* The @extra_bits can only be EXTENT_NORESERVE for now. */ + ASSERT(!(extra_bits & ~EXTENT_NORESERVE), "extra_bits=0x%x", extra_bits); + + /* Basic alignment check. */ + ASSERT(IS_ALIGNED(start, blocksize), "start=%llu blocksize=%u", + start, blocksize); + ASSERT(IS_ALIGNED(end + 1, blocksize), "inclusive end=%llu blocksize=%u", + end, blocksize); /* - * For experimental build, we error out instead of EAGAIN. - * - * We should not hit such out-of-band dirty folios anymore. + * Check and set DELALLOC_NEW flag, this needs to search tree thus can + * fail early. Thus we want to do this before clearing EXTENT_DELALLOC. */ - if (IS_ENABLED(CONFIG_BTRFS_EXPERIMENTAL)) { - DEBUG_WARN(); - btrfs_err_rl(fs_info, - "root %lld ino %llu folio %llu is marked dirty without notifying the fs", - btrfs_root_id(BTRFS_I(inode)->root), - btrfs_ino(BTRFS_I(inode)), - folio_pos(folio)); - return -EUCLEAN; + if (start >= i_size_read(&inode->vfs_inode) && + !(inode->flags & BTRFS_INODE_PREALLOC)) { + /* + * There can't be any extents following EOF in this case so just + * set the delalloc new bit for the range directly. + */ + extra_bits |= EXTENT_DELALLOC_NEW; + } else { + int ret; + + ret = btrfs_find_new_delalloc_bytes(inode, start, end + 1 - start, + NULL); + if (unlikely(ret)) + return ret; } - - /* - * folio_checked is set below when we create a fixup worker for this - * folio, don't try to create another one if we're already - * folio_test_checked. - * - * The extent_io writepage code will redirty the foio if we send back - * EAGAIN. - */ - if (folio_test_checked(folio)) - return -EAGAIN; - - fixup = kzalloc_obj(*fixup, GFP_NOFS); - if (!fixup) - return -EAGAIN; - - /* - * We are already holding a reference to this inode from - * write_cache_pages. We need to hold it because the space reservation - * takes place outside of the folio lock, and we can't trust - * folio->mapping outside of the folio lock. - */ - ihold(inode); - btrfs_folio_set_checked(fs_info, folio, folio_pos(folio), folio_size(folio)); - folio_get(folio); - btrfs_init_work(&fixup->work, btrfs_writepage_fixup_worker, NULL); - fixup->folio = folio; - fixup->inode = BTRFS_I(inode); - btrfs_queue_work(fs_info->fixup_workers, &fixup->work); - - return -EAGAIN; + /* Clear the old accounting as the range may already be dirty. */ + btrfs_clear_extent_bit(&inode->io_tree, start, end, + EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | + EXTENT_DEFRAG, cached_state); + return btrfs_set_extent_bit(&inode->io_tree, start, end, + EXTENT_DELALLOC | extra_bits, cached_state); } static int insert_reserved_file_extent(struct btrfs_trans_handle *trans, @@ -3798,7 +3621,7 @@ int btrfs_orphan_cleanup(struct btrfs_root *root) if (!inode && root == fs_info->tree_root) { struct btrfs_root *dead_root; - int is_dead_root = 0; + bool is_dead_root = false; /* * This is an orphan in the tree root. Currently these @@ -3820,7 +3643,7 @@ int btrfs_orphan_cleanup(struct btrfs_root *root) dead_root = radix_tree_lookup(&fs_info->fs_roots_radix, (unsigned long)found_key.objectid); if (dead_root && btrfs_root_refs(&dead_root->root_item) == 0) - is_dead_root = 1; + is_dead_root = true; spin_unlock(&fs_info->fs_roots_radix_lock); if (is_dead_root) { @@ -5181,12 +5004,7 @@ again: goto again; } - btrfs_clear_extent_bit(&inode->io_tree, block_start, block_end, - EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, - &cached_state); - - ret = btrfs_set_extent_delalloc(inode, block_start, block_end, 0, - &cached_state); + ret = btrfs_reset_extent_delalloc(inode, block_start, block_end, 0, &cached_state); if (ret) { btrfs_unlock_extent(io_tree, block_start, block_end, &cached_state); goto out_unlock; @@ -5211,8 +5029,6 @@ again: folio_zero_range(folio, zero_start - folio_pos(folio), zero_end - zero_start + 1); - btrfs_folio_clear_checked(fs_info, folio, block_start, - block_end + 1 - block_start); btrfs_folio_set_dirty(fs_info, folio, block_start, block_end + 1 - block_start); @@ -7657,12 +7473,6 @@ static int btrfs_migrate_folio(struct address_space *mapping, if (ret) return ret; - - if (folio_test_ordered(src)) { - folio_clear_ordered(src); - folio_set_ordered(dst); - } - return 0; } #else @@ -7751,18 +7561,20 @@ static void btrfs_invalidate_folio(struct folio *folio, size_t offset, page_end); ASSERT(range_end + 1 - cur < U32_MAX); range_len = range_end + 1 - cur; - if (!btrfs_folio_test_ordered(fs_info, folio, cur, range_len)) { - /* - * If Ordered is cleared, it means endio has - * already been executed for the range. - * We can't delete the extent states as - * btrfs_finish_ordered_io() may still use some of them. - */ + /* + * If the range is not dirty, the range has been submitted and + * since we have waited for the writeback, endio has been + * executed, thus we must skip the range to avoid double + * accounting for the ordered extent. + */ + if (!btrfs_folio_test_dirty(fs_info, folio, cur, range_len)) goto next; - } - btrfs_folio_clear_ordered(fs_info, folio, cur, range_len); /* + * The range is dirty meaning it has not been submitted. + * Here we need to truncate the OE range as the range will never + * be submitted. + * * IO on this page will never be started, so we need to account * for any ordered extents now. Don't clear EXTENT_DELALLOC_NEW * here, must leave that up for the ordered extent completion. @@ -7776,11 +7588,7 @@ static void btrfs_invalidate_folio(struct folio *folio, size_t offset, EXTENT_LOCKED | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, &cached_state); - spin_lock(&inode->ordered_tree_lock); - set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags); - ordered->truncated_len = min(ordered->truncated_len, - cur - ordered->file_offset); - spin_unlock(&inode->ordered_tree_lock); + btrfs_mark_ordered_extent_truncated(ordered, cur - ordered->file_offset); /* * If the ordered extent has finished, we're safe to delete all @@ -7823,13 +7631,8 @@ next: &cached_state); cur = range_end + 1; } - /* - * We have iterated through all ordered extents of the page, the page - * should not have Ordered anymore, or the above iteration - * did something wrong. - */ - ASSERT(!folio_test_ordered(folio)); - btrfs_folio_clear_checked(fs_info, folio, folio_pos(folio), folio_size(folio)); + btrfs_folio_clear_dirty(fs_info, folio, page_start, folio_size(folio)); + btrfs_clear_folio_dirty_tag(folio); if (!inode_evicting) __btrfs_release_folio(folio, GFP_NOFS); clear_folio_extent_mapped(folio); @@ -9687,7 +9490,7 @@ ssize_t btrfs_encoded_read_regular(struct kiocb *iocb, struct iov_iter *iter, pages = kzalloc_objs(struct page *, nr_pages, GFP_NOFS); if (!pages) return -ENOMEM; - ret = btrfs_alloc_page_array(nr_pages, pages, false); + ret = btrfs_alloc_page_array(nr_pages, pages, GFP_NOFS); if (ret) { ret = -ENOMEM; goto out; diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index a39460bf68a7..9d47d16394fc 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -56,6 +56,7 @@ #include "uuid-tree.h" #include "ioctl.h" #include "file.h" +#include "file-item.h" #include "scrub.h" #include "super.h" @@ -82,6 +83,30 @@ struct btrfs_ioctl_received_subvol_args_32 { #define BTRFS_IOC_SET_RECEIVED_SUBVOL_32 _IOWR(BTRFS_IOCTL_MAGIC, 37, \ struct btrfs_ioctl_received_subvol_args_32) + +struct btrfs_ioctl_get_subvol_info_args_32 { + __u64 treeid; + char name[BTRFS_VOL_NAME_MAX + 1]; + __u64 parent_id; + __u64 dirid; + __u64 generation; + __u64 flags; + __u8 uuid[BTRFS_UUID_SIZE]; + __u8 parent_uuid[BTRFS_UUID_SIZE]; + __u8 received_uuid[BTRFS_UUID_SIZE]; + __u64 ctransid; + __u64 otransid; + __u64 stransid; + __u64 rtransid; + struct btrfs_ioctl_timespec_32 ctime; + struct btrfs_ioctl_timespec_32 otime; + struct btrfs_ioctl_timespec_32 stime; + struct btrfs_ioctl_timespec_32 rtime; + __u64 reserved[8]; +} __attribute__ ((__packed__)); + +#define BTRFS_IOC_GET_SUBVOL_INFO_32 _IOR(BTRFS_IOCTL_MAGIC, 60, \ + struct btrfs_ioctl_get_subvol_info_args_32) #endif #if defined(CONFIG_64BIT) && defined(CONFIG_COMPAT) @@ -707,7 +732,7 @@ static int create_snapshot(struct btrfs_root *root, struct inode *dir, { struct btrfs_fs_info *fs_info = inode_to_fs_info(dir); struct inode *inode; - struct btrfs_pending_snapshot *pending_snapshot; + struct btrfs_pending_snapshot AUTO_KFREE(pending_snapshot); unsigned int trans_num_items; struct btrfs_trans_handle *trans; struct btrfs_block_rsv *block_rsv; @@ -816,7 +841,6 @@ free_pending: free_anon_bdev(pending_snapshot->anon_dev); kfree(pending_snapshot->root_item); btrfs_free_path(pending_snapshot->path); - kfree(pending_snapshot); return ret; } @@ -961,7 +985,7 @@ static noinline int btrfs_ioctl_resize(struct file *file, u64 new_size; u64 old_size; u64 devid = 1; - struct btrfs_ioctl_vol_args *vol_args; + struct btrfs_ioctl_vol_args AUTO_KFREE(vol_args); struct btrfs_device *device = NULL; char *sizestr; char *devstr = NULL; @@ -987,13 +1011,13 @@ static noinline int btrfs_ioctl_resize(struct file *file, } ret = btrfs_check_ioctl_vol_args_path(vol_args); if (ret < 0) - goto out_free; + goto out_drop; sizestr = vol_args->name; cancel = (strcmp("cancel", sizestr) == 0); ret = exclop_start_or_cancel_reloc(fs_info, BTRFS_EXCLOP_RESIZE, cancel); if (ret) - goto out_free; + goto out_drop; /* Exclusive operation is now claimed */ devstr = strchr(sizestr, ':'); @@ -1100,8 +1124,6 @@ static noinline int btrfs_ioctl_resize(struct file *file, old_size, new_size); out_finish: btrfs_exclop_finish(fs_info); -out_free: - kfree(vol_args); out_drop: mnt_drop_write_file(file); return ret; @@ -1114,7 +1136,7 @@ static noinline int __btrfs_ioctl_snap_create(struct file *file, struct btrfs_qgroup_inherit *inherit) { int ret; - struct qstr qname = QSTR_INIT(name, strlen(name)); + struct qstr qname = QSTR(name); if (!S_ISDIR(file_inode(file)->i_mode)) return -ENOTDIR; @@ -1179,7 +1201,7 @@ out_drop_write: static noinline int btrfs_ioctl_snap_create(struct file *file, void __user *arg, bool subvol) { - struct btrfs_ioctl_vol_args *vol_args; + struct btrfs_ioctl_vol_args AUTO_KFREE(vol_args); int ret; if (!S_ISDIR(file_inode(file)->i_mode)) @@ -1190,24 +1212,20 @@ static noinline int btrfs_ioctl_snap_create(struct file *file, return PTR_ERR(vol_args); ret = btrfs_check_ioctl_vol_args_path(vol_args); if (ret < 0) - goto out; + return ret; - ret = __btrfs_ioctl_snap_create(file, file_mnt_idmap(file), - vol_args->name, vol_args->fd, subvol, - false, NULL); - -out: - kfree(vol_args); - return ret; + return __btrfs_ioctl_snap_create(file, file_mnt_idmap(file), + vol_args->name, vol_args->fd, subvol, + false, NULL); } static noinline int btrfs_ioctl_snap_create_v2(struct file *file, void __user *arg, bool subvol) { - struct btrfs_ioctl_vol_args_v2 *vol_args; + struct btrfs_ioctl_vol_args_v2 AUTO_KFREE(vol_args); + struct btrfs_qgroup_inherit AUTO_KFREE(inherit); int ret; bool readonly = false; - struct btrfs_qgroup_inherit *inherit = NULL; if (!S_ISDIR(file_inode(file)->i_mode)) return -ENOTDIR; @@ -1217,44 +1235,32 @@ static noinline int btrfs_ioctl_snap_create_v2(struct file *file, return PTR_ERR(vol_args); ret = btrfs_check_ioctl_vol_args2_subvol_name(vol_args); if (ret < 0) - goto free_args; + return ret; - if (vol_args->flags & ~BTRFS_SUBVOL_CREATE_ARGS_MASK) { - ret = -EOPNOTSUPP; - goto free_args; - } + if (vol_args->flags & ~BTRFS_SUBVOL_CREATE_ARGS_MASK) + return -EOPNOTSUPP; if (vol_args->flags & BTRFS_SUBVOL_RDONLY) readonly = true; if (vol_args->flags & BTRFS_SUBVOL_QGROUP_INHERIT) { struct btrfs_fs_info *fs_info = inode_to_fs_info(file_inode(file)); - if (vol_args->size < sizeof(*inherit) || - vol_args->size > PAGE_SIZE) { - ret = -EINVAL; - goto free_args; - } + if (vol_args->size < sizeof(*inherit) || vol_args->size > PAGE_SIZE) + return -EINVAL; + inherit = memdup_user(vol_args->qgroup_inherit, vol_args->size); if (IS_ERR(inherit)) { - ret = PTR_ERR(inherit); - goto free_args; + return PTR_ERR(inherit); } ret = btrfs_qgroup_check_inherit(fs_info, inherit, vol_args->size); if (ret < 0) - goto free_inherit; + return ret; } - ret = __btrfs_ioctl_snap_create(file, file_mnt_idmap(file), - vol_args->name, vol_args->fd, subvol, - readonly, inherit); - if (ret) - goto free_inherit; -free_inherit: - kfree(inherit); -free_args: - kfree(vol_args); - return ret; + return __btrfs_ioctl_snap_create(file, file_mnt_idmap(file), + vol_args->name, vol_args->fd, subvol, + readonly, inherit); } static noinline int btrfs_ioctl_subvol_getflags(struct btrfs_inode *inode, @@ -1865,7 +1871,7 @@ out_put: static noinline int btrfs_ioctl_ino_lookup(struct btrfs_root *root, void __user *argp) { - struct btrfs_ioctl_ino_lookup_args *args; + struct btrfs_ioctl_ino_lookup_args AUTO_KFREE(args); int ret = 0; args = memdup_user(argp, sizeof(*args)); @@ -1895,9 +1901,8 @@ static noinline int btrfs_ioctl_ino_lookup(struct btrfs_root *root, out: if (ret == 0 && copy_to_user(argp, args, sizeof(*args))) - ret = -EFAULT; + return -EFAULT; - kfree(args); return ret; } @@ -1915,7 +1920,7 @@ out: */ static int btrfs_ioctl_ino_lookup_user(struct file *file, void __user *argp) { - struct btrfs_ioctl_ino_lookup_user_args *args; + struct btrfs_ioctl_ino_lookup_user_args AUTO_KFREE(args); struct inode *inode; int ret; @@ -1931,7 +1936,6 @@ static int btrfs_ioctl_ino_lookup_user(struct file *file, void __user *argp) * The subvolume does not exist under fd with which this is * called */ - kfree(args); return -EACCES; } @@ -1940,14 +1944,13 @@ static int btrfs_ioctl_ino_lookup_user(struct file *file, void __user *argp) if (ret == 0 && copy_to_user(argp, args, sizeof(*args))) ret = -EFAULT; - kfree(args); return ret; } /* Get the subvolume information in BTRFS_ROOT_ITEM and BTRFS_ROOT_BACKREF */ -static int btrfs_ioctl_get_subvol_info(struct inode *inode, void __user *argp) +static int _btrfs_ioctl_get_subvol_info(struct inode *inode, + struct btrfs_ioctl_get_subvol_info_args *subvol_info) { - struct btrfs_ioctl_get_subvol_info_args *subvol_info; struct btrfs_fs_info *fs_info; struct btrfs_root *root; struct btrfs_path *path; @@ -1956,7 +1959,6 @@ static int btrfs_ioctl_get_subvol_info(struct inode *inode, void __user *argp) struct btrfs_root_ref *rref; struct extent_buffer *leaf; unsigned long item_off; - unsigned long item_len; int slot; int ret = 0; @@ -1964,12 +1966,6 @@ static int btrfs_ioctl_get_subvol_info(struct inode *inode, void __user *argp) if (!path) return -ENOMEM; - subvol_info = kzalloc_obj(*subvol_info); - if (!subvol_info) { - btrfs_free_path(path); - return -ENOMEM; - } - fs_info = BTRFS_I(inode)->root->fs_info; /* Get root_item of inode's subvolume */ @@ -2031,33 +2027,91 @@ static int btrfs_ioctl_get_subvol_info(struct inode *inode, void __user *argp) btrfs_item_key_to_cpu(leaf, &key, slot); if (key.objectid == subvol_info->treeid && key.type == BTRFS_ROOT_BACKREF_KEY) { + u16 name_len; + subvol_info->parent_id = key.offset; rref = btrfs_item_ptr(leaf, slot, struct btrfs_root_ref); + name_len = btrfs_root_ref_name_len(leaf, rref); subvol_info->dirid = btrfs_root_ref_dirid(leaf, rref); - item_off = btrfs_item_ptr_offset(leaf, slot) - + sizeof(struct btrfs_root_ref); - item_len = btrfs_item_size(leaf, slot) - - sizeof(struct btrfs_root_ref); + item_off = btrfs_item_ptr_offset(leaf, slot) + sizeof(*rref); read_extent_buffer(leaf, subvol_info->name, - item_off, item_len); + item_off, name_len); } else { ret = -ENOENT; goto out; } } - btrfs_free_path(path); - path = NULL; - if (copy_to_user(argp, subvol_info, sizeof(*subvol_info))) - ret = -EFAULT; - out: btrfs_put_root(root); out_free: btrfs_free_path(path); - kfree(subvol_info); + return ret; +} + +#ifdef CONFIG_64BIT +static int btrfs_ioctl_get_subvol_info_32(struct inode *inode, void __user *argp) +{ + struct btrfs_ioctl_get_subvol_info_args AUTO_KFREE(subvol_info); + struct btrfs_ioctl_get_subvol_info_args_32 AUTO_KFREE(subvol_info_32); + int ret; + + subvol_info = kzalloc_obj(*subvol_info); + if (!subvol_info) + return -ENOMEM; + + subvol_info_32 = kzalloc_obj(*subvol_info_32); + if (!subvol_info_32) + return -ENOMEM; + + ret = _btrfs_ioctl_get_subvol_info(inode, subvol_info); + if (ret) + return ret; + + subvol_info_32->treeid = subvol_info->treeid; + memcpy(subvol_info_32->name, subvol_info->name, sizeof(subvol_info_32->name)); + subvol_info_32->parent_id = subvol_info->parent_id; + subvol_info_32->dirid = subvol_info->dirid; + subvol_info_32->generation = subvol_info->generation; + subvol_info_32->flags = subvol_info->flags; + memcpy(subvol_info_32->uuid, subvol_info->uuid, BTRFS_UUID_SIZE); + memcpy(subvol_info_32->parent_uuid, subvol_info->parent_uuid, BTRFS_UUID_SIZE); + memcpy(subvol_info_32->received_uuid, subvol_info->received_uuid, BTRFS_UUID_SIZE); + subvol_info_32->ctransid = subvol_info->ctransid; + subvol_info_32->otransid = subvol_info->otransid; + subvol_info_32->stransid = subvol_info->stransid; + subvol_info_32->rtransid = subvol_info->rtransid; + subvol_info_32->ctime.sec = subvol_info->ctime.sec; + subvol_info_32->ctime.nsec = subvol_info->ctime.nsec; + subvol_info_32->otime.sec = subvol_info->otime.sec; + subvol_info_32->otime.nsec = subvol_info->otime.nsec; + subvol_info_32->stime.sec = subvol_info->stime.sec; + subvol_info_32->stime.nsec = subvol_info->stime.nsec; + subvol_info_32->rtime.sec = subvol_info->rtime.sec; + subvol_info_32->rtime.nsec = subvol_info->rtime.nsec; + + if (copy_to_user(argp, subvol_info_32, sizeof(*subvol_info_32))) + ret = -EFAULT; + + return ret; +} +#endif + +static int btrfs_ioctl_get_subvol_info(struct inode *inode, void __user *argp) +{ + struct btrfs_ioctl_get_subvol_info_args AUTO_KFREE(subvol_info); + int ret; + + subvol_info = kzalloc_obj(*subvol_info); + if (!subvol_info) + return -ENOMEM; + + ret = _btrfs_ioctl_get_subvol_info(inode, subvol_info); + if (!ret && copy_to_user(argp, subvol_info, sizeof(*subvol_info))) + ret = -EFAULT; + return ret; } @@ -2068,7 +2122,7 @@ out_free: static int btrfs_ioctl_get_subvol_rootref(struct btrfs_root *root, void __user *argp) { - struct btrfs_ioctl_get_subvol_rootref_args *rootrefs; + struct btrfs_ioctl_get_subvol_rootref_args AUTO_KFREE(rootrefs); struct btrfs_root_ref *rref; struct btrfs_path *path; struct btrfs_key key; @@ -2151,8 +2205,6 @@ out: ret = -EFAULT; } - kfree(rootrefs); - return ret; } @@ -2167,8 +2219,8 @@ static noinline int btrfs_ioctl_snap_destroy(struct file *file, struct inode *inode; struct btrfs_root *root = BTRFS_I(dir)->root; struct btrfs_root *dest = NULL; - struct btrfs_ioctl_vol_args *vol_args = NULL; - struct btrfs_ioctl_vol_args_v2 *vol_args2 = NULL; + struct btrfs_ioctl_vol_args AUTO_KFREE(vol_args); + struct btrfs_ioctl_vol_args_v2 AUTO_KFREE(vol_args2); struct mnt_idmap *idmap = file_mnt_idmap(file); char *subvol_name, *subvol_name_ptr = NULL; int ret = 0; @@ -2186,10 +2238,8 @@ static noinline int btrfs_ioctl_snap_destroy(struct file *file, if (IS_ERR(vol_args2)) return PTR_ERR(vol_args2); - if (vol_args2->flags & ~BTRFS_SUBVOL_DELETE_ARGS_MASK) { - ret = -EOPNOTSUPP; - goto out; - } + if (vol_args2->flags & ~BTRFS_SUBVOL_DELETE_ARGS_MASK) + return -EOPNOTSUPP; /* * If SPEC_BY_ID is not set, we are looking for the subvolume by @@ -2198,23 +2248,21 @@ static noinline int btrfs_ioctl_snap_destroy(struct file *file, if (!(vol_args2->flags & BTRFS_SUBVOL_SPEC_BY_ID)) { ret = btrfs_check_ioctl_vol_args2_subvol_name(vol_args2); if (ret < 0) - goto out; + return ret; subvol_name = vol_args2->name; ret = mnt_want_write_file(file); if (ret) - goto out; + return ret; } else { struct inode *old_dir; - if (vol_args2->subvolid < BTRFS_FIRST_FREE_OBJECTID) { - ret = -EINVAL; - goto out; - } + if (vol_args2->subvolid < BTRFS_FIRST_FREE_OBJECTID) + return -EINVAL; ret = mnt_want_write_file(file); if (ret) - goto out; + return ret; dentry = btrfs_get_dentry(fs_info->sb, BTRFS_FIRST_FREE_OBJECTID, @@ -2284,13 +2332,13 @@ static noinline int btrfs_ioctl_snap_destroy(struct file *file, ret = btrfs_check_ioctl_vol_args_path(vol_args); if (ret < 0) - goto out; + return ret; subvol_name = vol_args->name; ret = mnt_want_write_file(file); if (ret) - goto out; + return ret; } if (strchr(subvol_name, '/') || @@ -2371,9 +2419,6 @@ free_parent: dput(parent); out_drop_write: mnt_drop_write_file(file); -out: - kfree(vol_args2); - kfree(vol_args); return ret; } @@ -2461,7 +2506,7 @@ out: static long btrfs_ioctl_add_dev(struct btrfs_fs_info *fs_info, void __user *arg) { - struct btrfs_ioctl_vol_args *vol_args; + struct btrfs_ioctl_vol_args AUTO_KFREE(vol_args); bool restore_op = false; int ret; @@ -2501,15 +2546,13 @@ static long btrfs_ioctl_add_dev(struct btrfs_fs_info *fs_info, void __user *arg) ret = btrfs_check_ioctl_vol_args_path(vol_args); if (ret < 0) - goto out_free; + goto out; ret = btrfs_init_new_device(fs_info, vol_args->name); if (!ret) btrfs_info(fs_info, "disk added %s", vol_args->name); -out_free: - kfree(vol_args); out: if (restore_op) btrfs_exclop_balance(fs_info, BTRFS_EXCLOP_BALANCE_PAUSED); @@ -2523,7 +2566,7 @@ static long btrfs_ioctl_rm_dev_v2(struct file *file, void __user *arg) BTRFS_DEV_LOOKUP_ARGS(args); struct inode *inode = file_inode(file); struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); - struct btrfs_ioctl_vol_args_v2 *vol_args; + struct btrfs_ioctl_vol_args_v2 AUTO_KFREE(vol_args); struct file *bdev_file = NULL; int ret; bool cancel = false; @@ -2582,7 +2625,6 @@ err_drop: bdev_fput(bdev_file); out: btrfs_put_dev_args_from_path(&args); - kfree(vol_args); return ret; } @@ -2591,7 +2633,7 @@ static long btrfs_ioctl_rm_dev(struct file *file, void __user *arg) BTRFS_DEV_LOOKUP_ARGS(args); struct inode *inode = file_inode(file); struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); - struct btrfs_ioctl_vol_args *vol_args; + struct btrfs_ioctl_vol_args AUTO_KFREE(vol_args); struct file *bdev_file = NULL; int ret; bool cancel = false; @@ -2605,7 +2647,7 @@ static long btrfs_ioctl_rm_dev(struct file *file, void __user *arg) ret = btrfs_check_ioctl_vol_args_path(vol_args); if (ret < 0) - goto out_free; + return ret; if (!strcmp("cancel", vol_args->name)) { cancel = true; @@ -2633,19 +2675,16 @@ static long btrfs_ioctl_rm_dev(struct file *file, void __user *arg) bdev_fput(bdev_file); out: btrfs_put_dev_args_from_path(&args); -out_free: - kfree(vol_args); return ret; } static long btrfs_ioctl_fs_info(const struct btrfs_fs_info *fs_info, void __user *arg) { - struct btrfs_ioctl_fs_info_args *fi_args; + struct btrfs_ioctl_fs_info_args AUTO_KFREE(fi_args); struct btrfs_device *device; struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; u64 flags_in; - int ret = 0; fi_args = memdup_user(arg, sizeof(*fi_args)); if (IS_ERR(fi_args)) @@ -2686,17 +2725,16 @@ static long btrfs_ioctl_fs_info(const struct btrfs_fs_info *fs_info, } if (copy_to_user(arg, fi_args, sizeof(*fi_args))) - ret = -EFAULT; + return -EFAULT; - kfree(fi_args); - return ret; + return 0; } static long btrfs_ioctl_dev_info(const struct btrfs_fs_info *fs_info, void __user *arg) { BTRFS_DEV_LOOKUP_ARGS(args); - struct btrfs_ioctl_dev_info_args *di_args; + struct btrfs_ioctl_dev_info_args AUTO_KFREE(di_args); struct btrfs_device *dev; int ret = 0; @@ -2730,7 +2768,6 @@ out: if (ret == 0 && copy_to_user(arg, di_args, sizeof(*di_args))) ret = -EFAULT; - kfree(di_args); return ret; } @@ -2792,9 +2829,13 @@ static long btrfs_ioctl_default_subvol(struct file *file, void __user *argp) if (IS_ERR_OR_NULL(di)) { btrfs_release_path(path); btrfs_end_transaction(trans); + if (di) + ret = PTR_ERR(di); + else + ret = -ENOENT; btrfs_err(fs_info, - "Umm, you don't have the default diritem, this isn't going to work"); - ret = -ENOENT; + "could not find default diritem for dir %llu: %d", + dir_id, ret); goto out_free; } @@ -3011,7 +3052,7 @@ static noinline long btrfs_ioctl_wait_sync(struct btrfs_fs_info *fs_info, static long btrfs_ioctl_scrub(struct file *file, void __user *arg) { struct btrfs_fs_info *fs_info = inode_to_fs_info(file_inode(file)); - struct btrfs_ioctl_scrub_args *sa; + struct btrfs_ioctl_scrub_args AUTO_KFREE(sa); int ret; if (!capable(CAP_SYS_ADMIN)) @@ -3026,15 +3067,13 @@ static long btrfs_ioctl_scrub(struct file *file, void __user *arg) if (IS_ERR(sa)) return PTR_ERR(sa); - if (sa->flags & ~BTRFS_SCRUB_SUPPORTED_FLAGS) { - ret = -EOPNOTSUPP; - goto out; - } + if (sa->flags & ~BTRFS_SCRUB_SUPPORTED_FLAGS) + return -EOPNOTSUPP; if (!(sa->flags & BTRFS_SCRUB_READONLY)) { ret = mnt_want_write_file(file); if (ret) - goto out; + return ret; } ret = btrfs_scrub_dev(fs_info, sa->devid, sa->start, sa->end, @@ -3058,8 +3097,7 @@ static long btrfs_ioctl_scrub(struct file *file, void __user *arg) if (!(sa->flags & BTRFS_SCRUB_READONLY)) mnt_drop_write_file(file); -out: - kfree(sa); + return ret; } @@ -3074,7 +3112,7 @@ static long btrfs_ioctl_scrub_cancel(struct btrfs_fs_info *fs_info) static long btrfs_ioctl_scrub_progress(struct btrfs_fs_info *fs_info, void __user *arg) { - struct btrfs_ioctl_scrub_args *sa; + struct btrfs_ioctl_scrub_args AUTO_KFREE(sa); int ret; if (!capable(CAP_SYS_ADMIN)) @@ -3087,40 +3125,36 @@ static long btrfs_ioctl_scrub_progress(struct btrfs_fs_info *fs_info, ret = btrfs_scrub_progress(fs_info, sa->devid, &sa->progress); if (ret == 0 && copy_to_user(arg, sa, sizeof(*sa))) - ret = -EFAULT; + return -EFAULT; - kfree(sa); return ret; } static long btrfs_ioctl_get_dev_stats(struct btrfs_fs_info *fs_info, void __user *arg) { - struct btrfs_ioctl_get_dev_stats *sa; + struct btrfs_ioctl_get_dev_stats AUTO_KFREE(sa); int ret; sa = memdup_user(arg, sizeof(*sa)); if (IS_ERR(sa)) return PTR_ERR(sa); - if ((sa->flags & BTRFS_DEV_STATS_RESET) && !capable(CAP_SYS_ADMIN)) { - kfree(sa); + if ((sa->flags & BTRFS_DEV_STATS_RESET) && !capable(CAP_SYS_ADMIN)) return -EPERM; - } ret = btrfs_get_dev_stats(fs_info, sa); if (ret == 0 && copy_to_user(arg, sa, sizeof(*sa))) - ret = -EFAULT; + return -EFAULT; - kfree(sa); return ret; } static long btrfs_ioctl_dev_replace(struct btrfs_fs_info *fs_info, void __user *arg) { - struct btrfs_ioctl_dev_replace_args *p; + struct btrfs_ioctl_dev_replace_args AUTO_KFREE(p); int ret; if (!capable(CAP_SYS_ADMIN)) @@ -3137,10 +3171,8 @@ static long btrfs_ioctl_dev_replace(struct btrfs_fs_info *fs_info, switch (p->cmd) { case BTRFS_IOCTL_DEV_REPLACE_CMD_START: - if (sb_rdonly(fs_info->sb)) { - ret = -EROFS; - goto out; - } + if (sb_rdonly(fs_info->sb)) + return -EROFS; if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_DEV_REPLACE)) { ret = BTRFS_ERROR_DEV_EXCL_RUN_IN_PROGRESS; } else { @@ -3162,9 +3194,8 @@ static long btrfs_ioctl_dev_replace(struct btrfs_fs_info *fs_info, } if ((ret == 0 || ret == -ECANCELED) && copy_to_user(arg, p, sizeof(*p))) - ret = -EFAULT; -out: - kfree(p); + return -EFAULT; + return ret; } @@ -3174,7 +3205,7 @@ static long btrfs_ioctl_ino_to_path(struct btrfs_root *root, void __user *arg) int i; u64 rel_ptr; int size; - struct btrfs_ioctl_ino_path_args *ipa = NULL; + struct btrfs_ioctl_ino_path_args AUTO_KFREE(ipa); struct inode_fs_paths *ipath __free(inode_fs_paths) = NULL; struct btrfs_path *path; @@ -3223,7 +3254,6 @@ static long btrfs_ioctl_ino_to_path(struct btrfs_root *root, void __user *arg) out: btrfs_free_path(path); - kfree(ipa); return ret; } @@ -3233,8 +3263,8 @@ static long btrfs_ioctl_logical_to_ino(struct btrfs_fs_info *fs_info, { int ret = 0; int size; - struct btrfs_ioctl_logical_ino_args *loi; - struct btrfs_data_container *inodes = NULL; + struct btrfs_ioctl_logical_ino_args AUTO_KFREE(loi); + struct btrfs_data_container AUTO_KVFREE(inodes); bool ignore_offset; if (!capable(CAP_SYS_ADMIN)) @@ -3249,41 +3279,32 @@ static long btrfs_ioctl_logical_to_ino(struct btrfs_fs_info *fs_info, size = min_t(u32, loi->size, SZ_64K); } else { /* All reserved bits must be 0 for now */ - if (memchr_inv(loi->reserved, 0, sizeof(loi->reserved))) { - ret = -EINVAL; - goto out_loi; - } + if (memchr_inv(loi->reserved, 0, sizeof(loi->reserved))) + return -EINVAL; + /* Only accept flags we have defined so far */ - if (loi->flags & ~(BTRFS_LOGICAL_INO_ARGS_IGNORE_OFFSET)) { - ret = -EINVAL; - goto out_loi; - } + if (loi->flags & ~(BTRFS_LOGICAL_INO_ARGS_IGNORE_OFFSET)) + return -EINVAL; + ignore_offset = loi->flags & BTRFS_LOGICAL_INO_ARGS_IGNORE_OFFSET; size = min_t(u32, loi->size, SZ_16M); } inodes = init_data_container(size); - if (IS_ERR(inodes)) { - ret = PTR_ERR(inodes); - goto out_loi; - } + if (IS_ERR(inodes)) + return PTR_ERR(inodes); ret = iterate_inodes_from_logical(loi->logical, fs_info, inodes, ignore_offset); if (ret == -EINVAL) - ret = -ENOENT; + return -ENOENT; if (ret < 0) - goto out; + return ret; ret = copy_to_user((void __user *)(unsigned long)loi->inodes, inodes, size); if (ret) ret = -EFAULT; -out: - kvfree(inodes); -out_loi: - kfree(loi); - return ret; } @@ -3380,7 +3401,7 @@ static long btrfs_ioctl_balance(struct file *file, void __user *arg) { struct btrfs_root *root = BTRFS_I(file_inode(file))->root; struct btrfs_fs_info *fs_info = root->fs_info; - struct btrfs_ioctl_balance_args *bargs; + struct btrfs_ioctl_balance_args AUTO_KFREE(bargs); struct btrfs_balance_control *bctl; bool need_unlock = true; int ret; @@ -3465,7 +3486,6 @@ out_unlock: btrfs_exclop_finish(fs_info); out: mnt_drop_write_file(file); - kfree(bargs); return ret; } @@ -3518,7 +3538,7 @@ static long btrfs_ioctl_quota_ctl(struct file *file, void __user *arg) { struct inode *inode = file_inode(file); struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); - struct btrfs_ioctl_quota_ctl_args *sa; + struct btrfs_ioctl_quota_ctl_args AUTO_KFREE(sa); int ret; if (!capable(CAP_SYS_ADMIN)) @@ -3577,7 +3597,6 @@ static long btrfs_ioctl_quota_ctl(struct file *file, void __user *arg) break; } - kfree(sa); drop_write: mnt_drop_write_file(file); return ret; @@ -3588,8 +3607,8 @@ static long btrfs_ioctl_qgroup_assign(struct file *file, void __user *arg) struct inode *inode = file_inode(file); struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); struct btrfs_root *root = BTRFS_I(inode)->root; - struct btrfs_ioctl_qgroup_assign_args *sa; - struct btrfs_qgroup_list *prealloc = NULL; + struct btrfs_ioctl_qgroup_assign_args AUTO_KFREE(sa); + struct btrfs_qgroup_list AUTO_KFREE(prealloc); struct btrfs_trans_handle *trans; int ret; int err; @@ -3614,7 +3633,7 @@ static long btrfs_ioctl_qgroup_assign(struct file *file, void __user *arg) prealloc = kzalloc_obj(*prealloc); if (!prealloc) { ret = -ENOMEM; - goto out; + goto drop_write; } } @@ -3622,7 +3641,7 @@ static long btrfs_ioctl_qgroup_assign(struct file *file, void __user *arg) trans = btrfs_start_transaction(root, 2); if (IS_ERR(trans)) { ret = PTR_ERR(trans); - goto out; + goto drop_write; } /* @@ -3648,9 +3667,6 @@ static long btrfs_ioctl_qgroup_assign(struct file *file, void __user *arg) if (err && !ret) ret = err; -out: - kfree(prealloc); - kfree(sa); drop_write: mnt_drop_write_file(file); return ret; @@ -3660,7 +3676,7 @@ static long btrfs_ioctl_qgroup_create(struct file *file, void __user *arg) { struct inode *inode = file_inode(file); struct btrfs_root *root = BTRFS_I(inode)->root; - struct btrfs_ioctl_qgroup_create_args *sa; + struct btrfs_ioctl_qgroup_create_args AUTO_KFREE(sa); struct btrfs_trans_handle *trans; int ret; int err; @@ -3683,12 +3699,12 @@ static long btrfs_ioctl_qgroup_create(struct file *file, void __user *arg) if (!sa->qgroupid) { ret = -EINVAL; - goto out; + goto drop_write; } if (sa->create && btrfs_is_fstree(sa->qgroupid)) { ret = -EINVAL; - goto out; + goto drop_write; } /* @@ -3698,7 +3714,7 @@ static long btrfs_ioctl_qgroup_create(struct file *file, void __user *arg) trans = btrfs_start_transaction(root, 2); if (IS_ERR(trans)) { ret = PTR_ERR(trans); - goto out; + goto drop_write; } if (sa->create) { @@ -3711,8 +3727,6 @@ static long btrfs_ioctl_qgroup_create(struct file *file, void __user *arg) if (err && !ret) ret = err; -out: - kfree(sa); drop_write: mnt_drop_write_file(file); return ret; @@ -3722,7 +3736,7 @@ static long btrfs_ioctl_qgroup_limit(struct file *file, void __user *arg) { struct inode *inode = file_inode(file); struct btrfs_root *root = BTRFS_I(inode)->root; - struct btrfs_ioctl_qgroup_limit_args *sa; + struct btrfs_ioctl_qgroup_limit_args AUTO_KFREE(sa); struct btrfs_trans_handle *trans; int ret; int err; @@ -3748,7 +3762,7 @@ static long btrfs_ioctl_qgroup_limit(struct file *file, void __user *arg) trans = btrfs_start_transaction(root, 1); if (IS_ERR(trans)) { ret = PTR_ERR(trans); - goto out; + goto drop_write; } qgroupid = sa->qgroupid; @@ -3763,8 +3777,6 @@ static long btrfs_ioctl_qgroup_limit(struct file *file, void __user *arg) if (err && !ret) ret = err; -out: - kfree(sa); drop_write: mnt_drop_write_file(file); return ret; @@ -3774,7 +3786,7 @@ static long btrfs_ioctl_quota_rescan(struct file *file, void __user *arg) { struct inode *inode = file_inode(file); struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); - struct btrfs_ioctl_quota_rescan_args *qsa; + struct btrfs_ioctl_quota_rescan_args AUTO_KFREE(qsa); int ret; if (!capable(CAP_SYS_ADMIN)) @@ -3795,13 +3807,11 @@ static long btrfs_ioctl_quota_rescan(struct file *file, void __user *arg) if (qsa->flags) { ret = -EINVAL; - goto out; + goto drop_write; } ret = btrfs_qgroup_rescan(fs_info); -out: - kfree(qsa); drop_write: mnt_drop_write_file(file); return ret; @@ -3946,8 +3956,8 @@ out: static long btrfs_ioctl_set_received_subvol_32(struct file *file, void __user *arg) { - struct btrfs_ioctl_received_subvol_args_32 *args32 = NULL; - struct btrfs_ioctl_received_subvol_args *args64 = NULL; + struct btrfs_ioctl_received_subvol_args_32 AUTO_KFREE(args32); + struct btrfs_ioctl_received_subvol_args AUTO_KFREE(args64); int ret = 0; args32 = memdup_user(arg, sizeof(*args32)); @@ -3955,10 +3965,8 @@ static long btrfs_ioctl_set_received_subvol_32(struct file *file, return PTR_ERR(args32); args64 = kmalloc_obj(*args64); - if (!args64) { - ret = -ENOMEM; - goto out; - } + if (!args64) + return -ENOMEM; memcpy(args64->uuid, args32->uuid, BTRFS_UUID_SIZE); args64->stransid = args32->stransid; @@ -3971,7 +3979,7 @@ static long btrfs_ioctl_set_received_subvol_32(struct file *file, ret = _btrfs_ioctl_set_received_subvol(file, file_mnt_idmap(file), args64); if (ret) - goto out; + return ret; memcpy(args32->uuid, args64->uuid, BTRFS_UUID_SIZE); args32->stransid = args64->stransid; @@ -3984,19 +3992,16 @@ static long btrfs_ioctl_set_received_subvol_32(struct file *file, ret = copy_to_user(arg, args32, sizeof(*args32)); if (ret) - ret = -EFAULT; + return -EFAULT; -out: - kfree(args32); - kfree(args64); - return ret; + return 0; } #endif static long btrfs_ioctl_set_received_subvol(struct file *file, void __user *arg) { - struct btrfs_ioctl_received_subvol_args *sa = NULL; + struct btrfs_ioctl_received_subvol_args AUTO_KFREE(sa); int ret = 0; sa = memdup_user(arg, sizeof(*sa)); @@ -4004,17 +4009,14 @@ static long btrfs_ioctl_set_received_subvol(struct file *file, return PTR_ERR(sa); ret = _btrfs_ioctl_set_received_subvol(file, file_mnt_idmap(file), sa); - if (ret) - goto out; + return ret; ret = copy_to_user(arg, sa, sizeof(*sa)); if (ret) - ret = -EFAULT; + return -EFAULT; -out: - kfree(sa); - return ret; + return 0; } static int btrfs_ioctl_get_fslabel(struct btrfs_fs_info *fs_info, @@ -4254,11 +4256,11 @@ out_drop_write: static int _btrfs_ioctl_send(struct btrfs_root *root, void __user *argp, bool compat) { - struct btrfs_ioctl_send_args *arg; - int ret; + struct btrfs_ioctl_send_args AUTO_KFREE(arg); if (compat) { #if defined(CONFIG_64BIT) && defined(CONFIG_COMPAT) + int ret; struct btrfs_ioctl_send_args_32 args32 = { 0 }; ret = copy_from_user(&args32, argp, sizeof(args32)); @@ -4283,9 +4285,7 @@ static int _btrfs_ioctl_send(struct btrfs_root *root, void __user *argp, bool co if (IS_ERR(arg)) return PTR_ERR(arg); } - ret = btrfs_ioctl_send(root, arg); - kfree(arg); - return ret; + return btrfs_ioctl_send(root, arg); } static int btrfs_ioctl_encoded_read(struct file *file, void __user *argp, @@ -4621,7 +4621,7 @@ static int btrfs_uring_read_extent(struct kiocb *iocb, struct iov_iter *iter, pages = kzalloc_objs(struct page *, nr_pages, GFP_NOFS); if (!pages) return -ENOMEM; - ret = btrfs_alloc_page_array(nr_pages, pages, 0); + ret = btrfs_alloc_page_array(nr_pages, pages, GFP_NOFS); if (ret) { ret = -ENOMEM; goto out_fail; @@ -5092,7 +5092,8 @@ static int btrfs_ioctl_subvol_sync(struct btrfs_fs_info *fs_info, void __user *a return -ENOENT; wait_for_deletion = true; - ASSERT(root_flags & BTRFS_ROOT_SUBVOL_DEAD); + ASSERT(root_flags & BTRFS_ROOT_SUBVOL_DEAD, "root_flags=0x%llx", + root_flags); sched_ret = schedule_timeout_interruptible(HZ); /* Early wake up or error. */ if (sched_ret != 0) @@ -5140,6 +5141,342 @@ static int btrfs_ioctl_shutdown(struct btrfs_fs_info *fs_info, unsigned long arg return ret; } +#define GET_CSUMS_BUF_MAX SZ_16M + +static int copy_csums_to_user(struct btrfs_fs_info *fs_info, u64 disk_bytenr, + u64 len, u8 __user *buf) +{ + struct btrfs_root *csum_root; + struct btrfs_ordered_sum *sums; + LIST_HEAD(list); + const u32 csum_size = fs_info->csum_size; + int ret; + + csum_root = btrfs_csum_root(fs_info, disk_bytenr); + if (unlikely(!csum_root)) { + btrfs_err(fs_info, "missing csum root for extent at bytenr %llu", disk_bytenr); + return -EUCLEAN; + } + + ret = btrfs_lookup_csums_list(csum_root, disk_bytenr, + disk_bytenr + len - 1, &list, false); + if (ret < 0) + return ret; + + ret = 0; + while (!list_empty(&list)) { + u64 offset; + size_t copy_size; + + sums = list_first_entry(&list, struct btrfs_ordered_sum, list); + list_del(&sums->list); + + offset = ((sums->logical - disk_bytenr) >> fs_info->sectorsize_bits) * csum_size; + copy_size = (sums->len >> fs_info->sectorsize_bits) * csum_size; + + if (copy_to_user(buf + offset, sums->sums, copy_size)) { + kfree(sums); + ret = -EFAULT; + goto out; + } + + kfree(sums); + } + +out: + while (!list_empty(&list)) { + sums = list_first_entry(&list, struct btrfs_ordered_sum, list); + list_del(&sums->list); + kfree(sums); + } + return ret; +} + +static int btrfs_ioctl_get_csums(struct file *file, void __user *argp) +{ + struct inode *vfs_inode = file_inode(file); + struct btrfs_inode *inode = BTRFS_I(vfs_inode); + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct btrfs_root *root = inode->root; + struct btrfs_ioctl_get_csums_args args; + BTRFS_PATH_AUTO_FREE(path); + const u64 ino = btrfs_ino(inode); + const u32 csum_size = fs_info->csum_size; + u8 __user *ubuf; + u64 buf_limit; + u64 buf_used = 0; + u64 cur_offset; + u64 end_offset; + u64 prev_extent_end; + struct btrfs_key key; + int ret; + + if (!(file->f_mode & FMODE_READ)) + return -EBADF; + + if (!S_ISREG(vfs_inode->i_mode)) + return -EINVAL; + + if (copy_from_user(&args, argp, sizeof(args))) + return -EFAULT; + + if (!IS_ALIGNED(args.offset, fs_info->sectorsize) || + !IS_ALIGNED(args.length, fs_info->sectorsize)) + return -EINVAL; + if (args.length == 0) + return -EINVAL; + if (args.offset + args.length < args.offset) + return -EOVERFLOW; + if (args.flags != 0) + return -EINVAL; + if (args.buf_size < sizeof(struct btrfs_ioctl_get_csums_entry)) + return -EINVAL; + + buf_limit = min_t(u64, args.buf_size, GET_CSUMS_BUF_MAX); + ubuf = (u8 __user *)(argp + offsetof(struct btrfs_ioctl_get_csums_args, buf)); + + if (clear_user(ubuf, buf_limit)) + return -EFAULT; + + cur_offset = args.offset; + end_offset = args.offset + args.length; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + ret = btrfs_wait_ordered_range(inode, cur_offset, args.length); + if (ret) + return ret; + + ret = down_read_interruptible(&vfs_inode->i_rwsem); + if (ret) + return ret; + + ret = btrfs_wait_ordered_range(inode, cur_offset, args.length); + if (ret) + goto out_unlock; + + /* NODATASUM early exit. */ + if (inode->flags & BTRFS_INODE_NODATASUM) { + struct btrfs_ioctl_get_csums_entry entry = { + .offset = cur_offset, + .length = end_offset - cur_offset, + .type = BTRFS_GET_CSUMS_NODATASUM, + }; + + if (copy_to_user(ubuf, &entry, sizeof(entry))) { + ret = -EFAULT; + goto out_unlock; + } + + buf_used = sizeof(entry); + cur_offset = end_offset; + goto done; + } + + prev_extent_end = cur_offset; + + while (cur_offset < end_offset) { + struct btrfs_file_extent_item *ei; + struct extent_buffer *leaf; + struct btrfs_ioctl_get_csums_entry entry = { 0 }; + u64 extent_end; + u64 disk_bytenr = 0; + u64 extent_offset = 0; + u64 range_start, range_len; + u64 entry_csum_size; + u64 key_offset; + int extent_type; + u8 compression; + u8 encryption; + + /* Search for the extent at or before cur_offset. */ + key.objectid = ino; + key.type = BTRFS_EXTENT_DATA_KEY; + key.offset = cur_offset; + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) + goto out_unlock; + + if (ret > 0 && path->slots[0] > 0) { + btrfs_item_key_to_cpu(path->nodes[0], &key, + path->slots[0] - 1); + if (key.objectid == ino && key.type == BTRFS_EXTENT_DATA_KEY) { + path->slots[0]--; + if (btrfs_file_extent_end(path) <= cur_offset) + path->slots[0]++; + } + } + + if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) { + ret = btrfs_next_leaf(root, path); + if (ret < 0) + goto out_unlock; + if (ret > 0) { + ret = 0; + btrfs_release_path(path); + break; + } + } + + leaf = path->nodes[0]; + + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY) { + btrfs_release_path(path); + break; + } + + extent_end = btrfs_file_extent_end(path); + key_offset = key.offset; + + /* Read extent fields before releasing the path. */ + ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item); + extent_type = btrfs_file_extent_type(leaf, ei); + compression = btrfs_file_extent_compression(leaf, ei); + encryption = btrfs_file_extent_encryption(leaf, ei); + + if (extent_type != BTRFS_FILE_EXTENT_INLINE) { + disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei); + if (disk_bytenr && compression == BTRFS_COMPRESS_NONE) + extent_offset = btrfs_file_extent_offset(leaf, ei); + } + + btrfs_release_path(path); + + /* Implicit hole (NO_HOLES feature). */ + if (prev_extent_end < key_offset) { + u64 hole_end = min(key_offset, end_offset); + u64 hole_len = hole_end - prev_extent_end; + + if (prev_extent_end >= cur_offset) { + entry.offset = prev_extent_end; + entry.length = hole_len; + entry.type = BTRFS_GET_CSUMS_ZEROED; + + if (buf_used + sizeof(entry) > buf_limit) + goto done; + if (copy_to_user(ubuf + buf_used, &entry, sizeof(entry))) { + ret = -EFAULT; + goto out_unlock; + } + buf_used += sizeof(entry); + cur_offset = hole_end; + } + + if (key_offset >= end_offset) { + cur_offset = end_offset; + break; + } + } + + /* Clamp to our query range. */ + range_start = max(cur_offset, key_offset); + range_len = min(extent_end, end_offset) - range_start; + + entry.offset = range_start; + entry.length = range_len; + + if (extent_type == BTRFS_FILE_EXTENT_INLINE) { + entry.type = BTRFS_GET_CSUMS_INLINE; + if (compression != BTRFS_COMPRESS_NONE) + entry.type |= BTRFS_GET_CSUMS_COMPRESSED; + if (encryption != 0) + entry.type |= BTRFS_GET_CSUMS_ENCRYPTED; + entry_csum_size = 0; + } else if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) { + entry.type = BTRFS_GET_CSUMS_ZEROED; + entry_csum_size = 0; + } else { + /* BTRFS_FILE_EXTENT_REG */ + if (disk_bytenr == 0) { + /* Explicit hole. */ + entry.type = BTRFS_GET_CSUMS_ZEROED; + entry_csum_size = 0; + } else if (encryption != 0 || compression != BTRFS_COMPRESS_NONE) { + entry.type = 0; + if (encryption != 0) + entry.type |= BTRFS_GET_CSUMS_ENCRYPTED; + if (compression != BTRFS_COMPRESS_NONE) + entry.type |= BTRFS_GET_CSUMS_COMPRESSED; + entry_csum_size = 0; + } else { + entry.type = BTRFS_GET_CSUMS_HAS_CSUMS; + entry_csum_size = (range_len >> fs_info->sectorsize_bits) * csum_size; + } + } + + /* Check if this entry (+ csum data) fits in the buffer. */ + if (buf_used + sizeof(entry) + entry_csum_size > buf_limit) { + if (buf_used == 0) { + ret = -EOVERFLOW; + goto out_unlock; + } + goto done; + } + + if (copy_to_user(ubuf + buf_used, &entry, sizeof(entry))) { + ret = -EFAULT; + goto out_unlock; + } + buf_used += sizeof(entry); + + if (entry.type == BTRFS_GET_CSUMS_HAS_CSUMS) { + ret = copy_csums_to_user(fs_info, + disk_bytenr + extent_offset + (range_start - key_offset), + range_len, ubuf + buf_used); + if (ret) + goto out_unlock; + buf_used += entry_csum_size; + } + + cur_offset = range_start + range_len; + prev_extent_end = extent_end; + + if (fatal_signal_pending(current)) { + if (buf_used == 0) { + ret = -EINTR; + goto out_unlock; + } + goto done; + } + + cond_resched(); + } + + /* Handle trailing implicit hole. */ + if (cur_offset < end_offset) { + struct btrfs_ioctl_get_csums_entry entry = { + .offset = prev_extent_end, + .length = end_offset - prev_extent_end, + .type = BTRFS_GET_CSUMS_ZEROED, + }; + + if (buf_used + sizeof(entry) <= buf_limit) { + if (copy_to_user(ubuf + buf_used, &entry, sizeof(entry))) { + ret = -EFAULT; + goto out_unlock; + } + buf_used += sizeof(entry); + cur_offset = end_offset; + } + } + +done: + args.offset = cur_offset; + args.length = (cur_offset < end_offset) ? end_offset - cur_offset : 0; + args.buf_size = buf_used; + + if (copy_to_user(argp, &args, sizeof(args))) + ret = -EFAULT; + +out_unlock: + up_read(&vfs_inode->i_rwsem); + return ret; +} + long btrfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { @@ -5273,6 +5610,10 @@ long btrfs_ioctl(struct file *file, unsigned int return btrfs_ioctl_set_features(file, argp); case BTRFS_IOC_GET_SUBVOL_INFO: return btrfs_ioctl_get_subvol_info(inode, argp); +#ifdef CONFIG_64BIT + case BTRFS_IOC_GET_SUBVOL_INFO_32: + return btrfs_ioctl_get_subvol_info_32(inode, argp); +#endif case BTRFS_IOC_GET_SUBVOL_ROOTREF: return btrfs_ioctl_get_subvol_rootref(root, argp); case BTRFS_IOC_INO_LOOKUP_USER: @@ -5297,6 +5638,8 @@ long btrfs_ioctl(struct file *file, unsigned int return btrfs_ioctl_subvol_sync(fs_info, argp); case BTRFS_IOC_SHUTDOWN: return btrfs_ioctl_shutdown(fs_info, arg); + case BTRFS_IOC_GET_CSUMS: + return btrfs_ioctl_get_csums(file, argp); } return -ENOTTY; diff --git a/fs/btrfs/lzo.c b/fs/btrfs/lzo.c index 2de18c7b563a..6e4aa22853ab 100644 --- a/fs/btrfs/lzo.c +++ b/fs/btrfs/lzo.c @@ -491,6 +491,17 @@ int lzo_decompress_bio(struct list_head *ws, struct compressed_bio *cb) return -EIO; } + /* The segment must not extend beyond the compressed input. */ + if (unlikely(cur_in + seg_len > compressed_len)) { + struct btrfs_inode *inode = cb->bbio.inode; + + btrfs_err(fs_info, + "lzo segment overflows compressed input, root %llu inode %llu offset %llu cur_in %u len %u compressed len %u", + btrfs_root_id(inode->root), btrfs_ino(inode), + cb->start, cur_in, seg_len, compressed_len); + return -EUCLEAN; + } + /* Copy the compressed segment payload into workspace */ copy_compressed_segment(cb, &fi, &cur_folio_index, workspace->cbuf, seg_len, &cur_in); diff --git a/fs/btrfs/ordered-data.c b/fs/btrfs/ordered-data.c index e5a24b3ff95e..b32d4eabe0ab 100644 --- a/fs/btrfs/ordered-data.c +++ b/fs/btrfs/ordered-data.c @@ -157,7 +157,8 @@ static struct btrfs_ordered_extent *alloc_ordered_extent( ((1U << BTRFS_ORDERED_NOCOW) | (1U << BTRFS_ORDERED_PREALLOC))); /* Only one type flag can be set. */ - ASSERT(has_single_bit_set(flags & BTRFS_ORDERED_EXCLUSIVE_FLAGS)); + ASSERT(has_single_bit_set(flags & BTRFS_ORDERED_EXCLUSIVE_FLAGS), + "flags=0x%lx", flags); /* DIRECT cannot be set with COMPRESSED nor ENCODED. */ if (test_bit(BTRFS_ORDERED_DIRECT, &flags)) { @@ -302,7 +303,7 @@ struct btrfs_ordered_extent *btrfs_alloc_ordered_extent( { struct btrfs_ordered_extent *entry; - ASSERT((flags & ~BTRFS_ORDERED_TYPE_FLAGS) == 0); + ASSERT((flags & ~BTRFS_ORDERED_TYPE_FLAGS) == 0, "flags=0x%lx", flags); /* * For regular writes, we just use the members in @file_extent. @@ -357,6 +358,18 @@ void btrfs_mark_ordered_extent_error(struct btrfs_ordered_extent *ordered) mapping_set_error(ordered->inode->vfs_inode.i_mapping, -EIO); } +void btrfs_mark_ordered_extent_truncated(struct btrfs_ordered_extent *ordered, + u64 truncate_len) +{ + struct btrfs_inode *inode = ordered->inode; + + ASSERT(truncate_len <= ordered->num_bytes); + spin_lock(&inode->ordered_tree_lock); + set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags); + ordered->truncated_len = min(ordered->truncated_len, truncate_len); + spin_unlock(&inode->ordered_tree_lock); +} + static void finish_ordered_fn(struct btrfs_work *work) { struct btrfs_ordered_extent *ordered_extent; @@ -1238,7 +1251,7 @@ struct btrfs_ordered_extent *btrfs_split_ordered_extent( trace_btrfs_ordered_extent_split(inode, ordered); - ASSERT(!(flags & (1U << BTRFS_ORDERED_COMPRESSED))); + ASSERT(!(flags & (1U << BTRFS_ORDERED_COMPRESSED)), "flags=0x%lx", flags); /* * The entire bio must be covered by the ordered extent, but we can't @@ -1260,7 +1273,7 @@ struct btrfs_ordered_extent *btrfs_split_ordered_extent( } /* We cannot split partially completed ordered extents. */ if (ordered->bytes_left) { - ASSERT(!(flags & ~BTRFS_ORDERED_TYPE_FLAGS)); + ASSERT(!(flags & ~BTRFS_ORDERED_TYPE_FLAGS), "flags=0x%lx", flags); if (WARN_ON_ONCE(ordered->bytes_left != ordered->disk_num_bytes)) return ERR_PTR(-EINVAL); } @@ -1307,7 +1320,8 @@ struct btrfs_ordered_extent *btrfs_split_ordered_extent( ordered->ram_bytes -= len; if (test_bit(BTRFS_ORDERED_IO_DONE, &ordered->flags)) { - ASSERT(ordered->bytes_left == 0); + ASSERT(ordered->bytes_left == 0, "ordered->bytes_left=%llu", + ordered->bytes_left); new->bytes_left = 0; } else { ordered->bytes_left -= len; diff --git a/fs/btrfs/ordered-data.h b/fs/btrfs/ordered-data.h index 03e12380a2fd..8d5d5ba1e02f 100644 --- a/fs/btrfs/ordered-data.h +++ b/fs/btrfs/ordered-data.h @@ -226,6 +226,8 @@ bool btrfs_try_lock_ordered_range(struct btrfs_inode *inode, u64 start, u64 end, struct btrfs_ordered_extent *btrfs_split_ordered_extent( struct btrfs_ordered_extent *ordered, u64 len); void btrfs_mark_ordered_extent_error(struct btrfs_ordered_extent *ordered); +void btrfs_mark_ordered_extent_truncated(struct btrfs_ordered_extent *ordered, + u64 truncate_len); int __init ordered_data_init(void); void __cold ordered_data_exit(void); diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c index 6838faceb6d5..502fb4a55cb2 100644 --- a/fs/btrfs/qgroup.c +++ b/fs/btrfs/qgroup.c @@ -373,10 +373,9 @@ static bool squota_check_parent_usage(struct btrfs_fs_info *fs_info, struct btrf parent->excl_cmpr != excl_cmpr_sum || parent->rfer_cmpr != rfer_cmpr_sum); WARN(mismatch, - "parent squota qgroup %hu/%llu has mismatched usage from its %d members. " + "parent squota qgroup " BTRFS_QGROUP_FMT " has mismatched usage from its %d members. " "%llu %llu %llu %llu vs %llu %llu %llu %llu\n", - btrfs_qgroup_level(parent->qgroupid), - btrfs_qgroup_subvolid(parent->qgroupid), nr_members, parent->excl, + BTRFS_QGROUP_FMT_VALUE(parent), nr_members, parent->excl, parent->rfer, parent->excl_cmpr, parent->rfer_cmpr, excl_sum, rfer_sum, excl_cmpr_sum, rfer_cmpr_sum); return mismatch; @@ -652,9 +651,8 @@ bool btrfs_check_quota_leak(const struct btrfs_fs_info *fs_info) if (qgroup->rsv.values[i]) { ret = true; btrfs_warn(fs_info, - "qgroup %hu/%llu has unreleased space, type %d rsv %llu", - btrfs_qgroup_level(qgroup->qgroupid), - btrfs_qgroup_subvolid(qgroup->qgroupid), + "qgroup " BTRFS_QGROUP_FMT " has unreleased space, type %d rsv %llu", + BTRFS_QGROUP_FMT_VALUE(qgroup), i, qgroup->rsv.values[i]); } } @@ -1858,14 +1856,13 @@ int btrfs_remove_qgroup(struct btrfs_trans_handle *trans, u64 qgroupid) * Thus its reserved space should all be zero, no matter if qgroup * is consistent or the mode. */ - if (qgroup->rsv.values[BTRFS_QGROUP_RSV_DATA] || - qgroup->rsv.values[BTRFS_QGROUP_RSV_META_PREALLOC] || - qgroup->rsv.values[BTRFS_QGROUP_RSV_META_PERTRANS]) { + if (unlikely(qgroup->rsv.values[BTRFS_QGROUP_RSV_DATA] || + qgroup->rsv.values[BTRFS_QGROUP_RSV_META_PREALLOC] || + qgroup->rsv.values[BTRFS_QGROUP_RSV_META_PERTRANS])) { DEBUG_WARN(); btrfs_warn_rl(fs_info, -"to be deleted qgroup %u/%llu has non-zero numbers, data %llu meta prealloc %llu meta pertrans %llu", - btrfs_qgroup_level(qgroup->qgroupid), - btrfs_qgroup_subvolid(qgroup->qgroupid), +"to be deleted qgroup " BTRFS_QGROUP_FMT " has non-zero numbers, data %llu meta prealloc %llu meta pertrans %llu", + BTRFS_QGROUP_FMT_VALUE(qgroup), qgroup->rsv.values[BTRFS_QGROUP_RSV_DATA], qgroup->rsv.values[BTRFS_QGROUP_RSV_META_PREALLOC], qgroup->rsv.values[BTRFS_QGROUP_RSV_META_PERTRANS]); @@ -1879,13 +1876,12 @@ int btrfs_remove_qgroup(struct btrfs_trans_handle *trans, u64 qgroupid) */ if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_FULL && !(fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT)) { - if (qgroup->rfer || qgroup->excl || - qgroup->rfer_cmpr || qgroup->excl_cmpr) { + if (unlikely(qgroup->rfer || qgroup->excl || + qgroup->rfer_cmpr || qgroup->excl_cmpr)) { DEBUG_WARN(); qgroup_mark_inconsistent(fs_info, - "to be deleted qgroup %u/%llu has non-zero numbers, rfer %llu rfer_cmpr %llu excl %llu excl_cmpr %llu", - btrfs_qgroup_level(qgroup->qgroupid), - btrfs_qgroup_subvolid(qgroup->qgroupid), +"to be deleted qgroup " BTRFS_QGROUP_FMT " has non-zero numbers, rfer %llu rfer_cmpr %llu excl %llu excl_cmpr %llu", + BTRFS_QGROUP_FMT_VALUE(qgroup), qgroup->rfer, qgroup->rfer_cmpr, qgroup->excl, qgroup->excl_cmpr); } @@ -4822,9 +4818,9 @@ int btrfs_qgroup_add_swapped_blocks(struct btrfs_root *subvol_root, entry = rb_entry(node, struct btrfs_qgroup_swapped_block, node); - if (entry->subvol_generation != block->subvol_generation || - entry->reloc_bytenr != block->reloc_bytenr || - entry->reloc_generation != block->reloc_generation) { + if (unlikely(entry->subvol_generation != block->subvol_generation || + entry->reloc_bytenr != block->reloc_bytenr || + entry->reloc_generation != block->reloc_generation)) { /* * Duplicated but mismatch entry found. Shouldn't happen. * Marking qgroup inconsistent should be enough for end @@ -4971,9 +4967,8 @@ int btrfs_record_squota_delta(struct btrfs_fs_info *fs_info, ASSERT(qg->excl == qg->rfer); if (WARN_ON_ONCE(sign < 0 && qg->excl < num_bytes)) { btrfs_warn(fs_info, - "squota underflow qg %hu/%llu excl %llu num_bytes %llu", - btrfs_qgroup_level(qg->qgroupid), - btrfs_qgroup_subvolid(qg->qgroupid), + "squota underflow qg " BTRFS_QGROUP_FMT " excl %llu num_bytes %llu", + BTRFS_QGROUP_FMT_VALUE(qg), qg->excl, num_bytes); qg->excl = 0; qg->rfer = 0; diff --git a/fs/btrfs/raid-stripe-tree.c b/fs/btrfs/raid-stripe-tree.c index 4b0186c83ad1..454a95bf542a 100644 --- a/fs/btrfs/raid-stripe-tree.c +++ b/fs/btrfs/raid-stripe-tree.c @@ -272,7 +272,9 @@ int btrfs_delete_raid_extent(struct btrfs_trans_handle *trans, u64 start, u64 le &key, key.offset - length, length); - ASSERT(key.offset - diff_end == length); + ASSERT(key.offset - diff_end == length, + "key.offset=%llu diff_end=%llu length=%llu", + key.offset, diff_end, length); break; } diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c index 08ee8f316d96..f7f7db40994c 100644 --- a/fs/btrfs/raid56.c +++ b/fs/btrfs/raid56.c @@ -466,7 +466,7 @@ static void __remove_rbio_from_cache(struct btrfs_raid_bio *rbio) int bucket = rbio_bucket(rbio); struct btrfs_stripe_hash_table *table; struct btrfs_stripe_hash *h; - int freeit = 0; + bool freeit = false; /* * check the bit again under the hash table lock. @@ -491,7 +491,7 @@ static void __remove_rbio_from_cache(struct btrfs_raid_bio *rbio) if (test_and_clear_bit(RBIO_CACHE_BIT, &rbio->flags)) { list_del_init(&rbio->stripe_cache); table->cache_size -= 1; - freeit = 1; + freeit = true; /* if the bio list isn't empty, this rbio is * still involved in an IO. We take it out @@ -855,7 +855,7 @@ static noinline void unlock_stripe(struct btrfs_raid_bio *rbio) { int bucket; struct btrfs_stripe_hash *h; - int keep_cache = 0; + bool keep_cache = false; bucket = rbio_bucket(rbio); h = rbio->bioc->fs_info->stripe_hash_table->table + bucket; @@ -874,7 +874,7 @@ static noinline void unlock_stripe(struct btrfs_raid_bio *rbio) */ if (list_empty(&rbio->plug_list) && test_bit(RBIO_CACHE_BIT, &rbio->flags)) { - keep_cache = 1; + keep_cache = true; clear_bit(RBIO_RMW_LOCKED_BIT, &rbio->flags); BUG_ON(!bio_list_empty(&rbio->bio_list)); goto done; @@ -1123,7 +1123,7 @@ static int alloc_rbio_pages(struct btrfs_raid_bio *rbio) { int ret; - ret = btrfs_alloc_page_array(rbio->nr_pages, rbio->stripe_pages, false); + ret = btrfs_alloc_page_array(rbio->nr_pages, rbio->stripe_pages, GFP_NOFS); if (ret < 0) return ret; /* Mapping all sectors */ @@ -1138,7 +1138,7 @@ static int alloc_rbio_parity_pages(struct btrfs_raid_bio *rbio) int ret; ret = btrfs_alloc_page_array(rbio->nr_pages - data_pages, - rbio->stripe_pages + data_pages, false); + rbio->stripe_pages + data_pages, GFP_NOFS); if (ret < 0) return ret; @@ -1732,7 +1732,7 @@ static int alloc_rbio_data_pages(struct btrfs_raid_bio *rbio) const int data_pages = rbio->nr_data * rbio->stripe_npages; int ret; - ret = btrfs_alloc_page_array(data_pages, rbio->stripe_pages, false); + ret = btrfs_alloc_page_array(data_pages, rbio->stripe_pages, GFP_NOFS); if (ret < 0) return ret; @@ -2695,7 +2695,7 @@ static int finish_parity_scrub(struct btrfs_raid_bio *rbio) phys_addr_t p_paddr = INVALID_PADDR; phys_addr_t q_paddr = INVALID_PADDR; struct bio_list bio_list; - int is_replace = 0; + bool is_replace = false; int ret; bio_list_init(&bio_list); @@ -2712,7 +2712,7 @@ static int finish_parity_scrub(struct btrfs_raid_bio *rbio) * need to duplicate the final write to replace target. */ if (bioc->replace_nr_stripes && bioc->replace_stripe_src == rbio->scrubp) { - is_replace = 1; + is_replace = true; bitmap_copy(pbitmap, &rbio->dbitmap, rbio->stripe_nsectors); } diff --git a/fs/btrfs/reflink.c b/fs/btrfs/reflink.c index 49865a463780..9a49d2ecb949 100644 --- a/fs/btrfs/reflink.c +++ b/fs/btrfs/reflink.c @@ -69,7 +69,8 @@ static int copy_inline_to_page(struct btrfs_inode *inode, struct address_space *mapping = inode->vfs_inode.i_mapping; int ret; - ASSERT(IS_ALIGNED(file_offset, block_size)); + ASSERT(IS_ALIGNED(file_offset, block_size), "file_offset=%llu block_size=%u", + file_offset, block_size); /* * We have flushed and locked the ranges of the source and destination @@ -94,9 +95,7 @@ static int copy_inline_to_page(struct btrfs_inode *inode, if (ret < 0) goto out_unlock; - btrfs_clear_extent_bit(&inode->io_tree, file_offset, range_end, - EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, NULL); - ret = btrfs_set_extent_delalloc(inode, file_offset, range_end, 0, NULL); + ret = btrfs_reset_extent_delalloc(inode, file_offset, range_end, 0, NULL); if (ret) goto out_unlock; @@ -141,7 +140,6 @@ static int copy_inline_to_page(struct btrfs_inode *inode, folio_zero_range(folio, datal, block_size - datal); btrfs_folio_set_uptodate(fs_info, folio, file_offset, block_size); - btrfs_folio_clear_checked(fs_info, folio, file_offset, block_size); btrfs_folio_set_dirty(fs_info, folio, file_offset, block_size); out_unlock: if (!IS_ERR(folio)) { @@ -181,10 +179,12 @@ static int clone_copy_inline_extent(struct btrfs_inode *inode, struct btrfs_drop_extents_args drop_args = { 0 }; int ret; struct btrfs_key key; + bool copied_inline_to_page = false; if (new_key->offset > 0) { ret = copy_inline_to_page(inode, new_key->offset, inline_data, size, datal, comp_type); + copied_inline_to_page = (ret == 0); goto out; } @@ -290,6 +290,60 @@ copy_inline_extent: btrfs_abort_transaction(trans, ret); out: if (!ret && !trans) { + if (copied_inline_to_page && + new_key->offset + datal > i_size_read(&inode->vfs_inode)) { + /* + * If we copied the inline extent data to a page/folio + * beyond the i_size of the destination inode, then we + * need to increase the i_size before we start a + * transaction to update the inode item. This is to + * prevent a deadlock when the flushoncommit mount + * option is used, which happens like this: + * + * 1) Task A clones an inline extent from inode X to an + * offset of inode Y that is beyond Y's current + * i_size. This means we copied the inline extent's + * data to a folio of inode Y that is beyond its EOF, + * using the call above to copy_inline_to_page(); + * + * 2) Task B starts a transaction commit and calls + * btrfs_start_delalloc_flush() to flush delalloc; + * + * 3) The delalloc flushing sees the new dirty folio of + * inode Y and when it attempts to flush it, it ends + * up at extent_writepage() and sees that the offset + * of the folio is beyond the i_size of inode Y, so + * it attempts to invalidate the folio by calling + * folio_invalidate(), which ends up at btrfs' folio + * invalidate callback - btrfs_invalidate_folio(). + * There it tries to lock the folio's range in inode + * Y's extent io tree, but it blocks since it's + * currently locked by task A - during reflink we + * lock the inodes and the source and destination + * ranges after flushing all delalloc and waiting for + * ordered extent completion - after that we don't + * expect to have dirty folios in the ranges, the + * exception is if we have to copy an inline extent's + * data (because the destination offset is not zero); + * + * 4) Task A then does the 'goto out' below and attempts + * to start a transaction to update the inode item, + * and then it's blocked since the current + * transaction is in the TRANS_STATE_COMMIT_START + * state. Therefore task A has to wait for the + * current transaction to become unblocked (its + * state >= TRANS_STATE_UNBLOCKED). + * + * This leads to a deadlock - the task committing the + * transaction waiting for the delalloc flushing which + * is blocked during folio invalidation on the inode's + * extent lock and the reflink task waiting for the + * current transaction to be unblocked so that it can + * start a new one to update the inode item (while + * holding the extent lock). + */ + i_size_write(&inode->vfs_inode, new_key->offset + datal); + } /* * No transaction here means we copied the inline extent into a * page of the destination inode. @@ -322,50 +376,7 @@ copy_to_page: ret = copy_inline_to_page(inode, new_key->offset, inline_data, size, datal, comp_type); - - /* - * If we copied the inline extent data to a page/folio beyond the i_size - * of the destination inode, then we need to increase the i_size before - * we start a transaction to update the inode item. This is to prevent a - * deadlock when the flushoncommit mount option is used, which happens - * like this: - * - * 1) Task A clones an inline extent from inode X to an offset of inode - * Y that is beyond Y's current i_size. This means we copied the - * inline extent's data to a folio of inode Y that is beyond its EOF, - * using the call above to copy_inline_to_page(); - * - * 2) Task B starts a transaction commit and calls - * btrfs_start_delalloc_flush() to flush delalloc; - * - * 3) The delalloc flushing sees the new dirty folio of inode Y and when - * it attempts to flush it, it ends up at extent_writepage() and sees - * that the offset of the folio is beyond the i_size of inode Y, so - * it attempts to invalidate the folio by calling folio_invalidate(), - * which ends up at btrfs' folio invalidate callback - - * btrfs_invalidate_folio(). There it tries to lock the folio's range - * in inode Y's extent io tree, but it blocks since it's currently - * locked by task A - during reflink we lock the inodes and the - * source and destination ranges after flushing all delalloc and - * waiting for ordered extent completion - after that we don't expect - * to have dirty folios in the ranges, the exception is if we have to - * copy an inline extent's data (because the destination offset is - * not zero); - * - * 4) Task A then does the 'goto out' below and attempts to start a - * transaction to update the inode item, and then it's blocked since - * the current transaction is in the TRANS_STATE_COMMIT_START state. - * Therefore task A has to wait for the current transaction to become - * unblocked (its state >= TRANS_STATE_UNBLOCKED). - * - * This leads to a deadlock - the task committing the transaction - * waiting for the delalloc flushing which is blocked during folio - * invalidation on the inode's extent lock and the reflink task waiting - * for the current transaction to be unblocked so that it can start a - * a new one to update the inode item (while holding the extent lock). - */ - if (ret == 0 && new_key->offset + datal > i_size_read(&inode->vfs_inode)) - i_size_write(&inode->vfs_inode, new_key->offset + datal); + copied_inline_to_page = (ret == 0); goto out; } @@ -459,7 +470,7 @@ process_slot: key.objectid != btrfs_ino(BTRFS_I(src))) break; - ASSERT(key.type == BTRFS_EXTENT_DATA_KEY); + ASSERT(key.type == BTRFS_EXTENT_DATA_KEY, "key.type=%u", key.type); extent = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item); diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 3ebaf5880125..fb85bc8b345c 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -178,8 +178,101 @@ struct reloc_control { bool create_reloc_tree; bool merge_reloc_tree; bool found_file_extent; + + refcount_t refs; }; +static struct reloc_control *get_reloc_control(struct btrfs_fs_info *fs_info) +{ + struct reloc_control *rc; + + /* Quick path, avoid lock contention on fs_info->reloc_ctl_lock. */ + if (!data_race(fs_info->reloc_ctl)) + return NULL; + + spin_lock(&fs_info->reloc_ctl_lock); + rc = fs_info->reloc_ctl; + if (rc) + refcount_inc(&rc->refs); + spin_unlock(&fs_info->reloc_ctl_lock); + + return rc; +} + +static void __del_reloc_root(struct btrfs_root *root); + +static noinline_for_stack void free_reloc_roots(struct list_head *list) +{ + struct btrfs_root *reloc_root, *tmp; + + list_for_each_entry_safe(reloc_root, tmp, list, root_list) + __del_reloc_root(reloc_root); +} + +static void put_reloc_control(struct reloc_control *rc) +{ + if (refcount_dec_and_test(&rc->refs)) { + struct mapping_node *node, *tmp; + + if (rc->extent_root) + ASSERT(rc->extent_root->fs_info->reloc_ctl != rc); + + free_reloc_roots(&rc->reloc_roots); + rbtree_postorder_for_each_entry_safe(node, tmp, + &rc->reloc_root_tree.rb_root, + rb_node) + kfree(node); + + if (rc->block_group) + btrfs_put_block_group(rc->block_group); + + kfree(rc); + } +} + +/* Helper to delete the 'address of tree root -> reloc tree' mapping. */ +static void __del_reloc_root(struct btrfs_root *root) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct rb_node *rb_node; + struct mapping_node AUTO_KFREE(node); + struct reloc_control *rc; + bool put_ref = false; + + rc = get_reloc_control(fs_info); + if (rc && root->node) { + spin_lock(&rc->reloc_root_tree.lock); + rb_node = rb_simple_search(&rc->reloc_root_tree.rb_root, + root->commit_root->start); + if (rb_node) { + node = rb_entry(rb_node, struct mapping_node, rb_node); + rb_erase(&node->rb_node, &rc->reloc_root_tree.rb_root); + RB_CLEAR_NODE(&node->rb_node); + } + spin_unlock(&rc->reloc_root_tree.lock); + ASSERT(!node || (struct btrfs_root *)node->data == root); + } + + /* + * We only put the reloc root here if it's on the list. There's a lot + * of places where the pattern is to splice the rc->reloc_roots, process + * the reloc roots, and then add the reloc root back onto + * rc->reloc_roots. If we call __del_reloc_root while it's off of the + * list we don't want the reference being dropped, because the guy + * messing with the list is in charge of the reference. + */ + spin_lock(&fs_info->trans_lock); + if (!list_empty(&root->root_list)) { + put_ref = true; + list_del_init(&root->root_list); + } + spin_unlock(&fs_info->trans_lock); + if (put_ref) + btrfs_put_root(root); + if (rc) + put_reloc_control(rc); +} + static void mark_block_processed(struct reloc_control *rc, struct btrfs_backref_node *node) { @@ -407,7 +500,7 @@ static noinline_for_stack struct btrfs_backref_node *build_backref_tree( struct reloc_control *rc, struct btrfs_key *node_key, int level, u64 bytenr) { - struct btrfs_backref_iter *iter; + struct btrfs_backref_iter iter; struct btrfs_backref_cache *cache = &rc->backref_cache; /* For searching parent of TREE_BLOCK_REF */ struct btrfs_path *path; @@ -416,9 +509,9 @@ static noinline_for_stack struct btrfs_backref_node *build_backref_tree( struct btrfs_backref_edge *edge; int ret; - iter = btrfs_backref_iter_alloc(rc->extent_root->fs_info); - if (!iter) - return ERR_PTR(-ENOMEM); + ret = btrfs_backref_iter_init(&iter); + if (ret < 0) + return ERR_PTR(ret); path = btrfs_alloc_path(); if (!path) { ret = -ENOMEM; @@ -435,7 +528,7 @@ static noinline_for_stack struct btrfs_backref_node *build_backref_tree( /* Breadth-first search to build backref cache */ do { - ret = btrfs_backref_add_tree_node(trans, cache, path, iter, + ret = btrfs_backref_add_tree_node(trans, cache, path, &iter, node_key, cur); if (ret < 0) goto out; @@ -460,8 +553,7 @@ static noinline_for_stack struct btrfs_backref_node *build_backref_tree( if (handle_useless_nodes(rc, node)) node = NULL; out: - btrfs_free_path(iter->path); - kfree(iter); + btrfs_free_path(iter.path); btrfs_free_path(path); if (ret) { btrfs_backref_error_cleanup(cache, node); @@ -476,12 +568,11 @@ out: /* * helper to add 'address of tree root -> reloc tree' mapping */ -static int __add_reloc_root(struct btrfs_root *root) +static int __add_reloc_root(struct btrfs_root *root, struct reloc_control *rc) { struct btrfs_fs_info *fs_info = root->fs_info; struct rb_node *rb_node; struct mapping_node *node; - struct reloc_control *rc = fs_info->reloc_ctl; node = kmalloc_obj(*node, GFP_NOFS); if (!node) @@ -504,49 +595,6 @@ static int __add_reloc_root(struct btrfs_root *root) return 0; } -/* - * helper to delete the 'address of tree root -> reloc tree' - * mapping - */ -static void __del_reloc_root(struct btrfs_root *root) -{ - struct btrfs_fs_info *fs_info = root->fs_info; - struct rb_node *rb_node; - struct mapping_node AUTO_KFREE(node); - struct reloc_control *rc = fs_info->reloc_ctl; - bool put_ref = false; - - if (rc && root->node) { - spin_lock(&rc->reloc_root_tree.lock); - rb_node = rb_simple_search(&rc->reloc_root_tree.rb_root, - root->commit_root->start); - if (rb_node) { - node = rb_entry(rb_node, struct mapping_node, rb_node); - rb_erase(&node->rb_node, &rc->reloc_root_tree.rb_root); - RB_CLEAR_NODE(&node->rb_node); - } - spin_unlock(&rc->reloc_root_tree.lock); - ASSERT(!node || (struct btrfs_root *)node->data == root); - } - - /* - * We only put the reloc root here if it's on the list. There's a lot - * of places where the pattern is to splice the rc->reloc_roots, process - * the reloc roots, and then add the reloc root back onto - * rc->reloc_roots. If we call __del_reloc_root while it's off of the - * list we don't want the reference being dropped, because the guy - * messing with the list is in charge of the reference. - */ - spin_lock(&fs_info->trans_lock); - if (!list_empty(&root->root_list)) { - put_ref = true; - list_del_init(&root->root_list); - } - spin_unlock(&fs_info->trans_lock); - if (put_ref) - btrfs_put_root(root); -} - /* * helper to update the 'address of tree root -> reloc tree' * mapping @@ -590,7 +638,7 @@ static struct btrfs_root *create_reloc_root(struct btrfs_trans_handle *trans, struct btrfs_key root_key; int ret = 0; - root_item = kmalloc(sizeof(*root_item), GFP_NOFS); + root_item = kmalloc_obj(*root_item, GFP_NOFS); if (!root_item) return ERR_PTR(-ENOMEM); @@ -700,11 +748,12 @@ int btrfs_init_reloc_root(struct btrfs_trans_handle *trans, { struct btrfs_fs_info *fs_info = root->fs_info; struct btrfs_root *reloc_root; - struct reloc_control *rc = fs_info->reloc_ctl; + struct reloc_control *rc; struct btrfs_block_rsv *rsv; - int clear_rsv = 0; - int ret; + bool clear_rsv = false; + int ret = 0; + rc = get_reloc_control(fs_info); if (!rc) return 0; @@ -713,7 +762,7 @@ int btrfs_init_reloc_root(struct btrfs_trans_handle *trans, * create/update the dead reloc tree */ if (reloc_root_is_dead(root)) - return 0; + goto out; /* * This is subtle but important. We do not do @@ -724,9 +773,8 @@ int btrfs_init_reloc_root(struct btrfs_trans_handle *trans, * in. */ if (root->reloc_root) { - reloc_root = root->reloc_root; - btrfs_set_root_last_trans(reloc_root, trans->transid); - return 0; + btrfs_set_root_last_trans(root->reloc_root, trans->transid); + goto out; } /* @@ -734,28 +782,33 @@ int btrfs_init_reloc_root(struct btrfs_trans_handle *trans, * reloc trees never need their own reloc tree. */ if (!rc->create_reloc_tree || btrfs_root_id(root) == BTRFS_TREE_RELOC_OBJECTID) - return 0; + goto out; if (!trans->reloc_reserved) { rsv = trans->block_rsv; trans->block_rsv = rc->block_rsv; - clear_rsv = 1; + clear_rsv = true; } reloc_root = create_reloc_root(trans, root, btrfs_root_id(root)); if (clear_rsv) trans->block_rsv = rsv; - if (IS_ERR(reloc_root)) - return PTR_ERR(reloc_root); + if (IS_ERR(reloc_root)) { + ret = PTR_ERR(reloc_root); + goto out; + } - ret = __add_reloc_root(reloc_root); + ret = __add_reloc_root(reloc_root, rc); ASSERT(ret != -EEXIST); if (ret) { /* Pairs with create_reloc_root */ btrfs_put_root(reloc_root); - return ret; + goto out; } root->reloc_root = btrfs_grab_root(reloc_root); - return 0; +out: + put_reloc_control(rc); + + return ret; } /* @@ -767,6 +820,7 @@ int btrfs_update_reloc_root(struct btrfs_trans_handle *trans, struct btrfs_fs_info *fs_info = root->fs_info; struct btrfs_root *reloc_root; struct btrfs_root_item *root_item; + struct reloc_control *rc; int ret; if (!have_reloc_root(root)) @@ -782,9 +836,9 @@ int btrfs_update_reloc_root(struct btrfs_trans_handle *trans, */ btrfs_grab_root(reloc_root); + rc = get_reloc_control(fs_info); /* root->reloc_root will stay until current relocation finished */ - if (fs_info->reloc_ctl && fs_info->reloc_ctl->merge_reloc_tree && - btrfs_root_refs(root_item) == 0) { + if (rc && rc->merge_reloc_tree && btrfs_root_refs(root_item) == 0) { set_bit(BTRFS_ROOT_DEAD_RELOC_TREE, &root->state); /* * Mark the tree as dead before we change reloc_root so @@ -804,6 +858,9 @@ int btrfs_update_reloc_root(struct btrfs_trans_handle *trans, ret = btrfs_update_root(trans, fs_info->tree_root, &reloc_root->root_key, root_item); btrfs_put_root(reloc_root); + if (rc) + put_reloc_control(rc); + return ret; } @@ -814,6 +871,7 @@ static int get_new_location(struct inode *reloc_inode, u64 *new_bytenr, u64 bytenr, u64 num_bytes) { struct btrfs_root *root = BTRFS_I(reloc_inode)->root; + struct btrfs_fs_info *fs_info = root->fs_info; BTRFS_PATH_AUTO_FREE(path); struct btrfs_file_extent_item *fi; struct extent_buffer *leaf; @@ -835,10 +893,23 @@ static int get_new_location(struct inode *reloc_inode, u64 *new_bytenr, fi = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item); - BUG_ON(btrfs_file_extent_offset(leaf, fi) || - btrfs_file_extent_compression(leaf, fi) || - btrfs_file_extent_encryption(leaf, fi) || - btrfs_file_extent_other_encoding(leaf, fi)); + /* + * The cluster-boundary key searched above is always written by + * relocation with offset 0: either by insert_prealloc_file_extent() + * (memsets the stack item to 0) or by the front portion of a partial + * writeback (offset=0 by construction). A non-zero value here means + * the on-disk leaf does not match what relocation wrote, i.e. + * corruption. The other encoding fields are caught earlier by + * tree-checker's check_extent_data_item(). + */ + if (unlikely(btrfs_file_extent_offset(leaf, fi))) { + btrfs_print_leaf(leaf); + btrfs_err(fs_info, +"unexpected non-zero offset in file extent item for data reloc inode %llu key offset %llu offset %llu", + btrfs_ino(BTRFS_I(reloc_inode)), bytenr, + btrfs_file_extent_offset(leaf, fi)); + return -EUCLEAN; + } if (num_bytes != btrfs_file_extent_disk_num_bytes(leaf, fi)) return -EINVAL; @@ -869,7 +940,7 @@ int replace_file_extents(struct btrfs_trans_handle *trans, u32 nritems; u32 i; int ret = 0; - int first = 1; + bool first = true; if (rc->stage != UPDATE_DATA_PTRS) return 0; @@ -907,7 +978,7 @@ int replace_file_extents(struct btrfs_trans_handle *trans, if (btrfs_root_id(root) != BTRFS_TREE_RELOC_OBJECTID) { if (first) { inode = btrfs_find_first_inode(root, key.objectid); - first = 0; + first = false; } else if (inode && btrfs_ino(inode) < key.objectid) { btrfs_add_delayed_iput(inode); inode = btrfs_find_first_inode(root, key.objectid); @@ -1021,7 +1092,7 @@ int replace_path(struct btrfs_trans_handle *trans, struct reloc_control *rc, u64 new_ptr_gen; u64 last_snapshot; u32 blocksize; - int cow = 0; + bool cow = false; int level; int ret; int slot; @@ -1127,7 +1198,7 @@ again: if (!cow) { btrfs_tree_unlock(parent); free_extent_buffer(parent); - cow = 1; + cow = true; goto again; } @@ -1515,7 +1586,7 @@ static noinline_for_stack int merge_reloc_root(struct reloc_control *rc, int reserve_level; int level; int max_level; - int replaced = 0; + bool replaced = false; int ret = 0; u32 min_reserved; @@ -1561,7 +1632,7 @@ static noinline_for_stack int merge_reloc_root(struct reloc_control *rc, * and * 2 since we have two trees to COW. */ reserve_level = max_t(int, 1, btrfs_root_level(root_item)); - min_reserved = fs_info->nodesize * reserve_level * 2; + min_reserved = (reserve_level << fs_info->nodesize_bits) * 2; memset(&next_key, 0, sizeof(next_key)); while (1) { @@ -1590,7 +1661,7 @@ static noinline_for_stack int merge_reloc_root(struct reloc_control *rc, btrfs_set_root_last_trans(reloc_root, trans->transid); trans->block_rsv = rc->block_rsv; - replaced = 0; + replaced = false; max_level = level; ret = walk_down_reloc_tree(reloc_root, path, &level); @@ -1612,7 +1683,7 @@ static noinline_for_stack int merge_reloc_root(struct reloc_control *rc, level = ret; btrfs_node_key_to_cpu(path->nodes[level], &key, path->slots[level]); - replaced = 1; + replaced = true; } ret = walk_up_reloc_tree(reloc_root, path, &level); @@ -1794,15 +1865,6 @@ again: return err; } -static noinline_for_stack -void free_reloc_roots(struct list_head *list) -{ - struct btrfs_root *reloc_root, *tmp; - - list_for_each_entry_safe(reloc_root, tmp, list, root_list) - __del_reloc_root(reloc_root); -} - static noinline_for_stack void merge_reloc_roots(struct reloc_control *rc) { @@ -1810,7 +1872,7 @@ void merge_reloc_roots(struct reloc_control *rc) struct btrfs_root *root; struct btrfs_root *reloc_root; LIST_HEAD(reloc_roots); - int found = 0; + bool found = false; int ret = 0; again: root = rc->extent_root; @@ -1826,7 +1888,7 @@ again: mutex_unlock(&fs_info->reloc_mutex); while (!list_empty(&reloc_roots)) { - found = 1; + found = true; reloc_root = list_first_entry(&reloc_roots, struct btrfs_root, root_list); root = btrfs_get_fs_root(fs_info, reloc_root->root_key.offset, @@ -1879,7 +1941,7 @@ again: } if (found) { - found = 0; + found = false; goto again; } out: @@ -1907,7 +1969,7 @@ out: * do the reloc_dirty_list afterwards. Meanwhile the root->reloc_root * will be cleaned up on unmount. * - * The remaining nodes will be cleaned up by free_reloc_control. + * The remaining nodes will be cleaned up by put_reloc_control(). */ } @@ -2559,7 +2621,7 @@ static int relocate_cowonly_block(struct btrfs_trans_handle *trans, nr_levels = max(btrfs_header_level(root->node) - block->level, 0) + 1; - num_bytes = fs_info->nodesize * nr_levels; + num_bytes = (nr_levels << fs_info->nodesize_bits); ret = refill_metadata_space(trans, rc, num_bytes); if (ret) { btrfs_put_root(root); @@ -2944,7 +3006,7 @@ static int relocate_file_extent_cluster(struct reloc_control *rc) if (!cluster->nr) return 0; - ra = kzalloc(sizeof(*ra), GFP_NOFS); + ra = kzalloc_obj(*ra, GFP_NOFS); if (!ra) return -ENOMEM; @@ -3187,13 +3249,12 @@ again: goto again; } } - if (ret) { + if (WARN_ON(ret)) { ASSERT(ret == 1); btrfs_print_leaf(path->nodes[0]); btrfs_err(fs_info, "tree block extent item (%llu) is not found in extent tree", bytenr); - WARN_ON(1); return -EINVAL; } @@ -3421,7 +3482,9 @@ static void set_reloc_control(struct reloc_control *rc) struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; mutex_lock(&fs_info->reloc_mutex); + spin_lock(&fs_info->reloc_ctl_lock); fs_info->reloc_ctl = rc; + spin_unlock(&fs_info->reloc_ctl_lock); mutex_unlock(&fs_info->reloc_mutex); } @@ -3430,7 +3493,9 @@ static void unset_reloc_control(struct reloc_control *rc) struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; mutex_lock(&fs_info->reloc_mutex); + spin_lock(&fs_info->reloc_ctl_lock); fs_info->reloc_ctl = NULL; + spin_unlock(&fs_info->reloc_ctl_lock); mutex_unlock(&fs_info->reloc_mutex); } @@ -3815,21 +3880,11 @@ static struct reloc_control *alloc_reloc_control(struct btrfs_fs_info *fs_info) rc->reloc_root_tree.rb_root = RB_ROOT; spin_lock_init(&rc->reloc_root_tree.lock); btrfs_extent_io_tree_init(fs_info, &rc->processed_blocks, IO_TREE_RELOC_BLOCKS); + refcount_set(&rc->refs, 1); + return rc; } -static void free_reloc_control(struct reloc_control *rc) -{ - struct mapping_node *node, *tmp; - - free_reloc_roots(&rc->reloc_roots); - rbtree_postorder_for_each_entry_safe(node, tmp, - &rc->reloc_root_tree.rb_root, rb_node) - kfree(node); - - kfree(rc); -} - /* * Print the block group being relocated */ @@ -3863,7 +3918,7 @@ static int add_remap_tree_entries(struct btrfs_trans_handle *trans, struct btrfs max_items = BTRFS_LEAF_DATA_SIZE(trans->fs_info) / sizeof(struct btrfs_item); - data_sizes = kzalloc(sizeof(u32) * min_t(u32, num_entries, max_items), GFP_NOFS); + data_sizes = kzalloc_objs(u32, min_t(u32, num_entries, max_items), GFP_NOFS); if (!data_sizes) return -ENOMEM; @@ -4038,7 +4093,7 @@ static int copy_remapped_data(struct btrfs_fs_info *fs_info, u64 old_addr, if (!pages) return -ENOMEM; - ret = btrfs_alloc_page_array(nr_pages, pages, 0); + ret = btrfs_alloc_page_array(nr_pages, pages, GFP_NOFS); if (ret) { ret = -ENOMEM; goto end; @@ -4454,7 +4509,7 @@ static int create_remap_tree_entries(struct btrfs_trans_handle *trans, btrfs_release_path(path); - space_runs = kmalloc(sizeof(*space_runs) * extent_count, GFP_NOFS); + space_runs = kmalloc_objs(*space_runs, extent_count, GFP_NOFS); if (!space_runs) { mutex_unlock(&bg->free_space_lock); return -ENOMEM; @@ -4543,7 +4598,7 @@ static int create_remap_tree_entries(struct btrfs_trans_handle *trans, mutex_unlock(&bg->free_space_lock); max_entries = extent_count + 2; - entries = kmalloc(sizeof(*entries) * max_entries, GFP_NOFS); + entries = kmalloc_objs(*entries, max_entries, GFP_NOFS); if (!entries) { ret = -ENOMEM; goto out; @@ -5367,12 +5422,13 @@ int btrfs_relocate_block_group(struct btrfs_fs_info *fs_info, u64 group_start, return -ENOMEM; } + rc->extent_root = extent_root; + /* Block group ref now owned by rc, put_reloc_control() will drop it. */ + rc->block_group = bg; + ret = reloc_chunk_start(fs_info); if (ret < 0) - goto out_put_bg; - - rc->extent_root = extent_root; - rc->block_group = bg; + goto out_put_rc; ret = btrfs_inc_block_group_ro(rc->block_group, true); if (ret) @@ -5441,9 +5497,8 @@ out: iput(rc->data_inode); btrfs_free_path(path); reloc_chunk_end(fs_info); -out_put_bg: - btrfs_put_block_group(bg); - free_reloc_control(rc); +out_put_rc: + put_reloc_control(rc); return ret; } @@ -5598,7 +5653,7 @@ int btrfs_recover_relocation(struct btrfs_fs_info *fs_info) goto out_unset; } - ret = __add_reloc_root(reloc_root); + ret = __add_reloc_root(reloc_root, rc); ASSERT(ret != -EEXIST); if (ret) { list_add_tail(&reloc_root->root_list, &reloc_roots); @@ -5632,7 +5687,7 @@ out_unset: unset_reloc_control(rc); reloc_chunk_end(fs_info); out_end: - free_reloc_control(rc); + put_reloc_control(rc); out: free_reloc_roots(&reloc_roots); @@ -5712,11 +5767,11 @@ int btrfs_reloc_cow_block(struct btrfs_trans_handle *trans, struct btrfs_fs_info *fs_info = root->fs_info; struct reloc_control *rc; struct btrfs_backref_node *node; - int first_cow = 0; + bool first_cow = false; int level; int ret = 0; - rc = fs_info->reloc_ctl; + rc = get_reloc_control(fs_info); if (!rc) return 0; @@ -5725,7 +5780,7 @@ int btrfs_reloc_cow_block(struct btrfs_trans_handle *trans, level = btrfs_header_level(buf); if (btrfs_header_generation(buf) <= btrfs_root_last_snapshot(&root->root_item)) - first_cow = 1; + first_cow = true; if (btrfs_root_id(root) == BTRFS_TREE_RELOC_OBJECTID && rc->create_reloc_tree) { WARN_ON(!first_cow && level == 0); @@ -5741,7 +5796,8 @@ int btrfs_reloc_cow_block(struct btrfs_trans_handle *trans, btrfs_err(fs_info, "bytenr %llu was found but our backref cache was expecting %llu or %llu", buf->start, node->bytenr, node->new_bytenr); - return -EUCLEAN; + ret = -EUCLEAN; + goto out; } btrfs_backref_drop_node_buffer(node); @@ -5764,6 +5820,9 @@ int btrfs_reloc_cow_block(struct btrfs_trans_handle *trans, if (level == 0 && first_cow && rc->stage == UPDATE_DATA_PTRS) ret = replace_file_extents(trans, rc, root, cow); +out: + put_reloc_control(rc); + return ret; } @@ -5812,13 +5871,16 @@ int btrfs_reloc_post_snapshot(struct btrfs_trans_handle *trans, struct btrfs_root *root = pending->root; struct btrfs_root *reloc_root; struct btrfs_root *new_root; - struct reloc_control *rc = root->fs_info->reloc_ctl; - int ret; + struct reloc_control *rc; + int ret = 0; - if (!rc || !have_reloc_root(root)) + rc = get_reloc_control(trans->fs_info); + if (!rc) return 0; - rc = root->fs_info->reloc_ctl; + if (!have_reloc_root(root)) + goto out; + rc->merging_rsv_size += rc->nodes_relocated; if (rc->merge_reloc_tree) { @@ -5826,23 +5888,28 @@ int btrfs_reloc_post_snapshot(struct btrfs_trans_handle *trans, rc->block_rsv, rc->nodes_relocated, true); if (ret) - return ret; + goto out; } new_root = pending->snap; reloc_root = create_reloc_root(trans, root->reloc_root, btrfs_root_id(new_root)); - if (IS_ERR(reloc_root)) - return PTR_ERR(reloc_root); + if (IS_ERR(reloc_root)) { + ret = PTR_ERR(reloc_root); + goto out; + } - ret = __add_reloc_root(reloc_root); + ret = __add_reloc_root(reloc_root, rc); ASSERT(ret != -EEXIST); if (ret) { /* Pairs with create_reloc_root */ btrfs_put_root(reloc_root); - return ret; + goto out; } new_root->reloc_root = btrfs_grab_root(reloc_root); - return 0; +out: + put_reloc_control(rc); + + return ret; } /* @@ -5850,14 +5917,15 @@ int btrfs_reloc_post_snapshot(struct btrfs_trans_handle *trans, * * Return U64_MAX if no running relocation. */ -u64 btrfs_get_reloc_bg_bytenr(const struct btrfs_fs_info *fs_info) +u64 btrfs_get_reloc_bg_bytenr(struct btrfs_fs_info *fs_info) { u64 logical = U64_MAX; - lockdep_assert_held(&fs_info->reloc_mutex); - + mutex_lock(&fs_info->reloc_mutex); if (fs_info->reloc_ctl && fs_info->reloc_ctl->block_group) logical = fs_info->reloc_ctl->block_group->start; + mutex_unlock(&fs_info->reloc_mutex); + return logical; } diff --git a/fs/btrfs/relocation.h b/fs/btrfs/relocation.h index d647823b5d13..bb7a86e7dbe3 100644 --- a/fs/btrfs/relocation.h +++ b/fs/btrfs/relocation.h @@ -41,7 +41,7 @@ int btrfs_reloc_post_snapshot(struct btrfs_trans_handle *trans, int btrfs_should_cancel_balance(const struct btrfs_fs_info *fs_info); struct btrfs_root *find_reloc_root(struct btrfs_fs_info *fs_info, u64 bytenr); bool btrfs_should_ignore_reloc_root(const struct btrfs_root *root); -u64 btrfs_get_reloc_bg_bytenr(const struct btrfs_fs_info *fs_info); +u64 btrfs_get_reloc_bg_bytenr(struct btrfs_fs_info *fs_info); int btrfs_translate_remap(struct btrfs_fs_info *fs_info, u64 *logical, u64 *length); int btrfs_remove_extent_from_remap_tree(struct btrfs_trans_handle *trans, struct btrfs_path *path, diff --git a/fs/btrfs/root-tree.c b/fs/btrfs/root-tree.c index d85a09ae1733..90659b287d90 100644 --- a/fs/btrfs/root-tree.c +++ b/fs/btrfs/root-tree.c @@ -27,20 +27,20 @@ static void btrfs_read_root_item(struct extent_buffer *eb, int slot, struct btrfs_root_item *item) { u32 len; - int need_reset = 0; + bool need_reset = false; len = btrfs_item_size(eb, slot); read_extent_buffer(eb, item, btrfs_item_ptr_offset(eb, slot), min_t(u32, len, sizeof(*item))); if (len < sizeof(*item)) - need_reset = 1; + need_reset = true; if (!need_reset && btrfs_root_generation(item) != btrfs_root_generation_v2(item)) { if (btrfs_root_generation_v2(item) != 0) { btrfs_warn(eb->fs_info, "mismatching generation and generation_v2 found in root item. This root was probably mounted with an older kernel. Resetting all new fields."); } - need_reset = 1; + need_reset = true; } if (need_reset) { /* Clear all members from generation_v2 onwards. */ diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index 1ac609239cbe..d2f7ac5b6e96 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -369,7 +369,8 @@ static int init_scrub_stripe(struct btrfs_fs_info *fs_info, ASSERT(BTRFS_STRIPE_LEN >> min_folio_shift <= SCRUB_STRIPE_MAX_FOLIOS); ret = btrfs_alloc_folio_array(BTRFS_STRIPE_LEN >> min_folio_shift, - fs_info->block_min_order, stripe->folios); + fs_info->block_min_order, stripe->folios, + GFP_NOFS); if (ret < 0) goto error; diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index 89d72d8cb85f..3ae480c7474b 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -17,6 +17,7 @@ #include #include #include +#include #include "send.h" #include "ctree.h" #include "backref.h" @@ -72,6 +73,8 @@ struct fs_path { #define FS_PATH_INLINE_SIZE \ sizeof_field(struct fs_path, inline_buf) +static void fs_path_free(struct fs_path *p); +DEFINE_FREE(fs_path_free, struct fs_path *, fs_path_free(_T)) /* reused for each extent */ struct clone_root { @@ -981,7 +984,7 @@ static int iterate_inode_ref(struct btrfs_root *root, struct btrfs_path *path, struct btrfs_inode_ref *iref; struct btrfs_inode_extref *extref; BTRFS_PATH_AUTO_FREE(tmp_path); - struct fs_path *p; + struct fs_path *p __free(fs_path_free) = NULL; u32 cur = 0; u32 total; int slot = path->slots[0]; @@ -998,11 +1001,8 @@ static int iterate_inode_ref(struct btrfs_root *root, struct btrfs_path *path, return -ENOMEM; tmp_path = alloc_path_for_send(); - if (!tmp_path) { - fs_path_free(p); + if (!tmp_path) return -ENOMEM; - } - if (found_key->type == BTRFS_INODE_REF_KEY) { ptr = (unsigned long)btrfs_item_ptr(eb, slot, @@ -1034,30 +1034,27 @@ static int iterate_inode_ref(struct btrfs_root *root, struct btrfs_path *path, start = btrfs_ref_to_path(root, tmp_path, name_len, name_off, eb, dir, p->buf, p->buf_len); - if (IS_ERR(start)) { - ret = PTR_ERR(start); - goto out; - } + if (IS_ERR(start)) + return PTR_ERR(start); + if (start < p->buf) { /* overflow , try again with larger buffer */ ret = fs_path_ensure_buf(p, p->buf_len + p->buf - start); if (ret < 0) - goto out; + return ret; start = btrfs_ref_to_path(root, tmp_path, name_len, name_off, eb, dir, p->buf, p->buf_len); - if (IS_ERR(start)) { - ret = PTR_ERR(start); - goto out; - } + if (IS_ERR(start)) + return PTR_ERR(start); + if (unlikely(start < p->buf)) { btrfs_err(root->fs_info, "send: path ref buffer underflow for key " BTRFS_KEY_FMT, BTRFS_KEY_FMT_VALUE(found_key)); - ret = -EINVAL; - goto out; + return -EINVAL; } } p->start = start; @@ -1065,17 +1062,15 @@ static int iterate_inode_ref(struct btrfs_root *root, struct btrfs_path *path, ret = fs_path_add_from_extent_buffer(p, eb, name_off, name_len); if (ret < 0) - goto out; + return ret; } cur += elem_size + name_len; ret = iterate(dir, p, ctx); if (ret) - goto out; + return ret; } -out: - fs_path_free(p); return ret; } @@ -2028,7 +2023,7 @@ static int is_first_ref(struct btrfs_root *root, const char *name, int name_len) { int ret; - struct fs_path *tmp_name; + struct fs_path *tmp_name __free(fs_path_free) = NULL; u64 tmp_dir; tmp_name = fs_path_alloc(); @@ -2037,17 +2032,13 @@ static int is_first_ref(struct btrfs_root *root, ret = get_first_ref(root, ino, &tmp_dir, NULL, tmp_name); if (ret < 0) - goto out; + return ret; - if (dir != tmp_dir || name_len != fs_path_len(tmp_name)) { - ret = 0; - goto out; - } + if (dir != tmp_dir || name_len != fs_path_len(tmp_name)) + return 0; ret = !memcmp(tmp_name->start, name, name_len); -out: - fs_path_free(tmp_name); return ret; } @@ -2196,13 +2187,13 @@ static int did_overwrite_ref(struct send_ctx *sctx, */ static int did_overwrite_first_ref(struct send_ctx *sctx, u64 ino, u64 gen) { - int ret = 0; - struct fs_path *name = NULL; + int ret; + struct fs_path *name __free(fs_path_free) = NULL; u64 dir; u64 dir_gen; if (!sctx->parent_root) - goto out; + return 0; name = fs_path_alloc(); if (!name) @@ -2210,14 +2201,10 @@ static int did_overwrite_first_ref(struct send_ctx *sctx, u64 ino, u64 gen) ret = get_first_ref(sctx->parent_root, ino, &dir, &dir_gen, name); if (ret < 0) - goto out; + return ret; - ret = did_overwrite_ref(sctx, dir, dir_gen, ino, gen, - name->start, fs_path_len(name)); - -out: - fs_path_free(name); - return ret; + return did_overwrite_ref(sctx, dir, dir_gen, ino, gen, + name->start, fs_path_len(name)); } static inline struct name_cache_entry *name_cache_search(struct send_ctx *sctx, @@ -2375,10 +2362,10 @@ static int get_cur_path(struct send_ctx *sctx, u64 ino, u64 gen, struct fs_path *dest) { int ret = 0; - struct fs_path *name = NULL; + struct fs_path *name __free(fs_path_free) = NULL; u64 parent_inode = 0; u64 parent_gen = 0; - int stop = 0; + bool stop = false; const bool is_cur_inode = (ino == sctx->cur_ino && gen == sctx->cur_inode_gen); if (is_cur_inode && fs_path_len(&sctx->cur_inode_path) > 0) { @@ -2389,10 +2376,8 @@ static int get_cur_path(struct send_ctx *sctx, u64 ino, u64 gen, } name = fs_path_alloc(); - if (!name) { - ret = -ENOMEM; - goto out; - } + if (!name) + return -ENOMEM; dest->reversed = 1; fs_path_reset(dest); @@ -2413,7 +2398,7 @@ static int get_cur_path(struct send_ctx *sctx, u64 ino, u64 gen, wdm = get_waiting_dir_move(sctx, ino); if (wdm && wdm->orphanized) { ret = gen_unique_name(sctx, ino, gen, name); - stop = 1; + stop = true; } else if (wdm) { ret = get_first_ref(sctx->parent_root, ino, &parent_inode, &parent_gen, name); @@ -2422,7 +2407,7 @@ static int get_cur_path(struct send_ctx *sctx, u64 ino, u64 gen, &parent_inode, &parent_gen, name); if (ret) - stop = 1; + stop = true; } if (ret < 0) @@ -2437,7 +2422,6 @@ static int get_cur_path(struct send_ctx *sctx, u64 ino, u64 gen, } out: - fs_path_free(name); if (!ret) { fs_path_unreverse(dest); if (is_cur_inode && dest != &sctx->cur_inode_path) @@ -2787,7 +2771,7 @@ static int trim_dir_utimes_cache(struct send_ctx *sctx) static int send_create_inode(struct send_ctx *sctx, u64 ino) { int ret = 0; - struct fs_path *p; + struct fs_path *p __free(fs_path_free) = NULL; int cmd; struct btrfs_inode_info info; u64 gen; @@ -2801,7 +2785,7 @@ static int send_create_inode(struct send_ctx *sctx, u64 ino) if (ino != sctx->cur_ino) { ret = get_inode_info(sctx->send_root, ino, &info); if (ret < 0) - goto out; + return ret; gen = info.gen; mode = info.mode; rdev = info.rdev; @@ -2826,17 +2810,16 @@ static int send_create_inode(struct send_ctx *sctx, u64 ino) } else { btrfs_warn(sctx->send_root->fs_info, "unexpected inode type %o", (int)(mode & S_IFMT)); - ret = -EOPNOTSUPP; - goto out; + return -EOPNOTSUPP; } ret = begin_cmd(sctx, cmd); if (ret < 0) - goto out; + return ret; ret = gen_unique_name(sctx, ino, gen, p); if (ret < 0) - goto out; + return ret; TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); TLV_PUT_U64(sctx, BTRFS_SEND_A_INO, ino); @@ -2845,7 +2828,7 @@ static int send_create_inode(struct send_ctx *sctx, u64 ino) fs_path_reset(p); ret = read_symlink(sctx->send_root, ino, p); if (ret < 0) - goto out; + return ret; TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH_LINK, p); } else if (S_ISCHR(mode) || S_ISBLK(mode) || S_ISFIFO(mode) || S_ISSOCK(mode)) { @@ -2855,12 +2838,9 @@ static int send_create_inode(struct send_ctx *sctx, u64 ino) ret = send_cmd(sctx); if (ret < 0) - goto out; - + return ret; tlv_put_failure: -out: - fs_path_free(p); return ret; } @@ -3039,7 +3019,7 @@ static int orphanize_inode(struct send_ctx *sctx, u64 ino, u64 gen, struct fs_path *path) { int ret; - struct fs_path *orphan; + struct fs_path *orphan __free(fs_path_free) = NULL; orphan = fs_path_alloc(); if (!orphan) @@ -3047,17 +3027,15 @@ static int orphanize_inode(struct send_ctx *sctx, u64 ino, u64 gen, ret = gen_unique_name(sctx, ino, gen, orphan); if (ret < 0) - goto out; + return ret; ret = send_rename(sctx, path, orphan); if (ret < 0) - goto out; + return ret; if (ino == sctx->cur_ino && gen == sctx->cur_inode_gen) ret = fs_path_copy(&sctx->cur_inode_path, orphan); -out: - fs_path_free(orphan); return ret; } @@ -3349,7 +3327,7 @@ static int add_pending_dir_move(struct send_ctx *sctx, struct rb_node *parent = NULL; struct pending_dir_move *entry = NULL, *pm; struct recorded_ref *cur; - int exists = 0; + bool exists = false; int ret; pm = kmalloc_obj(*pm); @@ -3370,7 +3348,7 @@ static int add_pending_dir_move(struct send_ctx *sctx, } else if (parent_ino > entry->parent_ino) { p = &(*p)->rb_right; } else { - exists = 1; + exists = true; break; } } @@ -3467,9 +3445,9 @@ static int path_loop(struct send_ctx *sctx, struct fs_path *name, static int apply_dir_move(struct send_ctx *sctx, struct pending_dir_move *pm) { - struct fs_path *from_path = NULL; - struct fs_path *to_path = NULL; - struct fs_path *name = NULL; + struct fs_path *from_path __free(fs_path_free) = NULL; + struct fs_path *to_path __free(fs_path_free) = NULL; + struct fs_path *name __free(fs_path_free) = NULL; u64 orig_progress = sctx->send_progress; struct recorded_ref *cur; u64 parent_ino, parent_gen; @@ -3482,10 +3460,8 @@ static int apply_dir_move(struct send_ctx *sctx, struct pending_dir_move *pm) name = fs_path_alloc(); from_path = fs_path_alloc(); - if (!name || !from_path) { - ret = -ENOMEM; - goto out; - } + if (!name || !from_path) + return -ENOMEM; dm = get_waiting_dir_move(sctx, pm->ino); ASSERT(dm); @@ -3599,9 +3575,6 @@ finish: } out: - fs_path_free(name); - fs_path_free(from_path); - fs_path_free(to_path); sctx->send_progress = orig_progress; return ret; @@ -4157,7 +4130,7 @@ static int rename_current_inode(struct send_ctx *sctx, /* * This does all the move/link/unlink/rmdir magic. */ -static int process_recorded_refs(struct send_ctx *sctx, int *pending_move) +static int process_recorded_refs(struct send_ctx *sctx, bool *pending_move) { struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; int ret = 0; @@ -4417,7 +4390,7 @@ static int process_recorded_refs(struct send_ctx *sctx, int *pending_move) goto out; if (ret == 1) { can_rename = false; - *pending_move = 1; + *pending_move = true; } } @@ -4428,7 +4401,7 @@ static int process_recorded_refs(struct send_ctx *sctx, int *pending_move) goto out; if (ret == 1) { can_rename = false; - *pending_move = 1; + *pending_move = true; } } @@ -4793,7 +4766,7 @@ static int process_all_refs(struct send_ctx *sctx, struct btrfs_key key; struct btrfs_key found_key; iterate_inode_ref_t cb; - int pending_move = 0; + bool pending_move = false; path = alloc_path_for_send(); if (!path) @@ -6524,8 +6497,7 @@ static int process_all_extents(struct send_ctx *sctx) } static int process_recorded_refs_if_needed(struct send_ctx *sctx, bool at_end, - int *pending_move, - int *refs_processed) + bool *pending_move, bool *refs_processed) { int ret; @@ -6543,7 +6515,7 @@ static int process_recorded_refs_if_needed(struct send_ctx *sctx, bool at_end, if (ret < 0) return ret; - *refs_processed = 1; + *refs_processed = true; return 0; } @@ -6559,12 +6531,12 @@ static int finish_inode_if_needed(struct send_ctx *sctx, bool at_end) u64 right_uid; u64 right_gid; u64 right_fileattr; - int need_chmod = 0; - int need_chown = 0; + bool need_chmod = false; + bool need_chown = false; bool need_fileattr = false; - int need_truncate = 1; - int pending_move = 0; - int refs_processed = 0; + bool need_truncate = true; + bool pending_move = false; + bool refs_processed = false; if (sctx->ignore_cur_inode) return 0; @@ -6602,11 +6574,11 @@ static int finish_inode_if_needed(struct send_ctx *sctx, bool at_end) left_fileattr = info.fileattr; if (!sctx->parent_root || sctx->cur_inode_new) { - need_chown = 1; + need_chown = true; if (!S_ISLNK(sctx->cur_inode_mode)) - need_chmod = 1; + need_chmod = true; if (sctx->cur_inode_next_write_offset == sctx->cur_inode_size) - need_truncate = 0; + need_truncate = false; } else { u64 old_size; @@ -6620,15 +6592,15 @@ static int finish_inode_if_needed(struct send_ctx *sctx, bool at_end) right_fileattr = info.fileattr; if (left_uid != right_uid || left_gid != right_gid) - need_chown = 1; + need_chown = true; if (!S_ISLNK(sctx->cur_inode_mode) && left_mode != right_mode) - need_chmod = 1; + need_chmod = true; if (!S_ISLNK(sctx->cur_inode_mode) && left_fileattr != right_fileattr) need_fileattr = true; if ((old_size == sctx->cur_inode_size) || (sctx->cur_inode_size > old_size && sctx->cur_inode_next_write_offset == sctx->cur_inode_size)) - need_truncate = 0; + need_truncate = false; } if (S_ISREG(sctx->cur_inode_mode)) { @@ -7986,7 +7958,7 @@ long btrfs_ioctl_send(struct btrfs_root *send_root, const struct btrfs_ioctl_sen u64 *clone_sources_tmp = NULL; int clone_sources_to_rollback = 0; size_t alloc_size; - int sort_clone_roots = 0; + bool sort_clone_roots = false; struct btrfs_lru_cache_entry *entry; struct btrfs_lru_cache_entry *tmp; @@ -8209,7 +8181,7 @@ long btrfs_ioctl_send(struct btrfs_root *send_root, const struct btrfs_ioctl_sen sort(sctx->clone_roots, sctx->clone_roots_cnt, sizeof(*sctx->clone_roots), __clone_root_cmp_sort, NULL); - sort_clone_roots = 1; + sort_clone_roots = true; ret = flush_delalloc_roots(sctx); if (ret) diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index f0436eea1544..e6641597b321 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -1411,6 +1411,13 @@ static void btrfs_preempt_reclaim_metadata_space(struct work_struct *work) * This is where we reclaim all of the pinned space generated by running the * iputs * + * RECLAIM_ZONES + * This state only works for the zoned mode. We scan the block groups in the + * reclaim_bgs_list and check if we can relocate them. If yes perform the + * relocation to garbage collect the zone. On each of these runs + * BTRFS_ZONED_SYNC_RECLAIM_BATCH (5) block-groups will be reclaimed, after all + * unused block-groups have been deleted. + * * RESET_ZONES * This state works only for the zoned mode. We scan the unused block group * list and reset the zones and reuse the block group. @@ -1698,6 +1705,7 @@ static int handle_reserve_ticket(struct btrfs_space_info *space_info, ARRAY_SIZE(evict_flush_states)); break; case BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE: + case BTRFS_RESERVE_FLUSH_ZONED_RELOCATION: priority_reclaim_data_space(space_info, ticket); break; default: @@ -1961,6 +1969,7 @@ int btrfs_reserve_data_bytes(struct btrfs_space_info *space_info, u64 bytes, ASSERT(flush == BTRFS_RESERVE_FLUSH_DATA || flush == BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE || + flush == BTRFS_RESERVE_FLUSH_ZONED_RELOCATION || flush == BTRFS_RESERVE_NO_FLUSH, "flush=%d", flush); ASSERT(!current->journal_info || flush != BTRFS_RESERVE_FLUSH_DATA, "current->journal_info=0x%lx flush=%d", diff --git a/fs/btrfs/space-info.h b/fs/btrfs/space-info.h index 24f45072ca4b..aa836e8a9d4a 100644 --- a/fs/btrfs/space-info.h +++ b/fs/btrfs/space-info.h @@ -77,6 +77,17 @@ enum btrfs_reserve_flush_enum { */ BTRFS_RESERVE_FLUSH_ALL_STEAL, + /* + * This is for relocation on zoned filesystems only. We need to use + * priority flushing for this, because otherwise we can deadlock on + * waiting for a ticket, that cannot be granted, because we cannot do + * any allocations. + * + * Apart from being specific to zoned relocation, it is equal to + * BTRFS_FLUSH_FREE_SPACE_INODE. + */ + BTRFS_RESERVE_FLUSH_ZONED_RELOCATION, + /* * This is for btrfs_use_block_rsv only. We have exhausted our block * rsv and our global block rsv. This can happen for things like diff --git a/fs/btrfs/subpage.c b/fs/btrfs/subpage.c index f82e71f5d88b..56060acac2e9 100644 --- a/fs/btrfs/subpage.c +++ b/fs/btrfs/subpage.c @@ -10,41 +10,13 @@ * * Limitations: * - * - Only support 64K page size for now - * This is to make metadata handling easier, as 64K page would ensure - * all nodesize would fit inside one page, thus we don't need to handle - * cases where a tree block crosses several pages. + * - Metadata must be fully aligned to node size + * So when nodesize <= page size, the metadata can never cross folio boundaries. * - * - Only metadata read-write for now - * The data read-write part is in development. - * - * - Metadata can't cross 64K page boundary - * btrfs-progs and kernel have done that for a while, thus only ancient - * filesystems could have such problem. For such case, do a graceful - * rejection. - * - * Special behavior: - * - * - Metadata - * Metadata read is fully supported. - * Meaning when reading one tree block will only trigger the read for the - * needed range, other unrelated range in the same page will not be touched. - * - * Metadata write support is partial. - * The writeback is still for the full page, but we will only submit - * the dirty extent buffers in the page. - * - * This means, if we have a metadata page like this: - * - * Page offset - * 0 16K 32K 48K 64K - * |/////////| |///////////| - * \- Tree block A \- Tree block B - * - * Even if we just want to writeback tree block A, we will also writeback - * tree block B if it's also dirty. - * - * This may cause extra metadata writeback which results more COW. + * - Only support blocks per folio <= min(BTRFS_MAX_FOLIO_SIZE / fs block size, + * BTRFS_MAX_BLOCKS_PER_FOLIO) + * This is to ensure we can afford an on-stack bitmap, without the need to allocate + * bitmap memory at runtime. * * Implementation: * @@ -224,11 +196,8 @@ static bool btrfs_subpage_end_and_test_lock(const struct btrfs_fs_info *fs_info, struct folio *folio, u64 start, u32 len) { struct btrfs_folio_state *bfs = folio_get_private(folio); - const int start_bit = subpage_calc_start_bit(fs_info, folio, locked, start, len); const int nbits = (len >> fs_info->sectorsize_bits); unsigned long flags; - unsigned int cleared = 0; - int bit = start_bit; bool last; btrfs_subpage_assert(fs_info, folio, start, len); @@ -245,15 +214,10 @@ static bool btrfs_subpage_end_and_test_lock(const struct btrfs_fs_info *fs_info, spin_unlock_irqrestore(&bfs->lock, flags); return true; } - - for_each_set_bit_from(bit, bfs->bitmaps, start_bit + nbits) { - clear_bit(bit, bfs->bitmaps); - cleared++; - } - ASSERT(atomic_read(&bfs->nr_locked) >= cleared, - "atomic_read(&bfs->nr_locked)=%d cleared=%d", - atomic_read(&bfs->nr_locked), cleared); - last = atomic_sub_and_test(cleared, &bfs->nr_locked); + ASSERT(atomic_read(&bfs->nr_locked) >= nbits, + "atomic_read(&bfs->nr_locked)=%d nbits=%d", + atomic_read(&bfs->nr_locked), nbits); + last = atomic_sub_and_test(nbits, &bfs->nr_locked); spin_unlock_irqrestore(&bfs->lock, flags); return last; } @@ -305,15 +269,13 @@ void btrfs_folio_end_lock(const struct btrfs_fs_info *fs_info, } void btrfs_folio_end_lock_bitmap(const struct btrfs_fs_info *fs_info, - struct folio *folio, unsigned long bitmap) + struct folio *folio, unsigned long *bitmap) { struct btrfs_folio_state *bfs = folio_get_private(folio); const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio); - const int start_bit = blocks_per_folio * btrfs_bitmap_nr_locked; + const unsigned int nbits = bitmap_weight(bitmap, blocks_per_folio); unsigned long flags; bool last = false; - int cleared = 0; - int bit; if (!btrfs_is_subpage(fs_info, folio)) { folio_unlock(folio); @@ -327,14 +289,10 @@ void btrfs_folio_end_lock_bitmap(const struct btrfs_fs_info *fs_info, } spin_lock_irqsave(&bfs->lock, flags); - for_each_set_bit(bit, &bitmap, blocks_per_folio) { - if (test_and_clear_bit(bit + start_bit, bfs->bitmaps)) - cleared++; - } - ASSERT(atomic_read(&bfs->nr_locked) >= cleared, - "atomic_read(&bfs->nr_locked)=%d cleared=%d", - atomic_read(&bfs->nr_locked), cleared); - last = atomic_sub_and_test(cleared, &bfs->nr_locked); + ASSERT(atomic_read(&bfs->nr_locked) >= nbits, + "atomic_read(&bfs->nr_locked)=%d nbits=%d", + atomic_read(&bfs->nr_locked), nbits); + last = atomic_sub_and_test(nbits, &bfs->nr_locked); spin_unlock_irqrestore(&bfs->lock, flags); if (last) folio_unlock(folio); @@ -479,64 +437,6 @@ void btrfs_subpage_clear_writeback(const struct btrfs_fs_info *fs_info, spin_unlock_irqrestore(&bfs->lock, flags); } -void btrfs_subpage_set_ordered(const struct btrfs_fs_info *fs_info, - struct folio *folio, u64 start, u32 len) -{ - struct btrfs_folio_state *bfs = folio_get_private(folio); - unsigned int start_bit = subpage_calc_start_bit(fs_info, folio, - ordered, start, len); - unsigned long flags; - - spin_lock_irqsave(&bfs->lock, flags); - bitmap_set(bfs->bitmaps, start_bit, len >> fs_info->sectorsize_bits); - folio_set_ordered(folio); - spin_unlock_irqrestore(&bfs->lock, flags); -} - -void btrfs_subpage_clear_ordered(const struct btrfs_fs_info *fs_info, - struct folio *folio, u64 start, u32 len) -{ - struct btrfs_folio_state *bfs = folio_get_private(folio); - unsigned int start_bit = subpage_calc_start_bit(fs_info, folio, - ordered, start, len); - unsigned long flags; - - spin_lock_irqsave(&bfs->lock, flags); - bitmap_clear(bfs->bitmaps, start_bit, len >> fs_info->sectorsize_bits); - if (subpage_test_bitmap_all_zero(fs_info, folio, ordered)) - folio_clear_ordered(folio); - spin_unlock_irqrestore(&bfs->lock, flags); -} - -void btrfs_subpage_set_checked(const struct btrfs_fs_info *fs_info, - struct folio *folio, u64 start, u32 len) -{ - struct btrfs_folio_state *bfs = folio_get_private(folio); - unsigned int start_bit = subpage_calc_start_bit(fs_info, folio, - checked, start, len); - unsigned long flags; - - spin_lock_irqsave(&bfs->lock, flags); - bitmap_set(bfs->bitmaps, start_bit, len >> fs_info->sectorsize_bits); - if (subpage_test_bitmap_all_set(fs_info, folio, checked)) - folio_set_checked(folio); - spin_unlock_irqrestore(&bfs->lock, flags); -} - -void btrfs_subpage_clear_checked(const struct btrfs_fs_info *fs_info, - struct folio *folio, u64 start, u32 len) -{ - struct btrfs_folio_state *bfs = folio_get_private(folio); - unsigned int start_bit = subpage_calc_start_bit(fs_info, folio, - checked, start, len); - unsigned long flags; - - spin_lock_irqsave(&bfs->lock, flags); - bitmap_clear(bfs->bitmaps, start_bit, len >> fs_info->sectorsize_bits); - folio_clear_checked(folio); - spin_unlock_irqrestore(&bfs->lock, flags); -} - /* * Unlike set/clear which is dependent on each page status, for test all bits * are tested in the same way. @@ -560,8 +460,6 @@ bool btrfs_subpage_test_##name(const struct btrfs_fs_info *fs_info, \ IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(uptodate); IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(dirty); IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(writeback); -IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(ordered); -IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(checked); /* * Note that, in selftests (extent-io-tests), we can have empty fs_info passed @@ -657,30 +555,55 @@ IMPLEMENT_BTRFS_PAGE_OPS(dirty, folio_mark_dirty, folio_clear_dirty_for_io, folio_test_dirty); IMPLEMENT_BTRFS_PAGE_OPS(writeback, folio_start_writeback, folio_end_writeback, folio_test_writeback); -IMPLEMENT_BTRFS_PAGE_OPS(ordered, folio_set_ordered, folio_clear_ordered, - folio_test_ordered); -IMPLEMENT_BTRFS_PAGE_OPS(checked, folio_set_checked, folio_clear_checked, - folio_test_checked); -#define GET_SUBPAGE_BITMAP(fs_info, folio, name, dst) \ -{ \ - const unsigned int __bpf = btrfs_blocks_per_folio(fs_info, folio); \ - const struct btrfs_folio_state *__bfs = folio_get_private(folio); \ - \ - ASSERT(__bpf <= BITS_PER_LONG); \ - *dst = bitmap_read(__bfs->bitmaps, \ - __bpf * btrfs_bitmap_nr_##name, __bpf); \ +#define DEFINE_GET_SUBPAGE_BITMAP(name) \ +static inline unsigned long get_bitmap_value_##name( \ + const struct btrfs_fs_info *fs_info, \ + struct folio *folio) \ +{ \ + const unsigned int __bpf = btrfs_blocks_per_folio(fs_info, folio); \ + const struct btrfs_folio_state *__bfs = folio_get_private(folio); \ + unsigned long value; \ + \ + ASSERT(__bpf <= BITS_PER_LONG); \ + value = bitmap_read(__bfs->bitmaps, __bpf * btrfs_bitmap_nr_##name, \ + __bpf); \ + return value; \ +} \ +static inline const unsigned long *get_bitmap_pointer_##name( \ + const struct btrfs_fs_info *fs_info, \ + struct folio *folio) \ +{ \ + const unsigned int __bpf = btrfs_blocks_per_folio(fs_info, folio); \ + struct btrfs_folio_state *__bfs = folio_get_private(folio); \ + unsigned long *pointer; \ + \ + ASSERT(__bpf >= BITS_PER_LONG); \ + ASSERT(IS_ALIGNED(__bpf, BITS_PER_LONG)); \ + pointer = __bfs->bitmaps + (BIT_WORD(__bpf) * btrfs_bitmap_nr_##name); \ + return pointer; \ } -#define SUBPAGE_DUMP_BITMAP(fs_info, folio, name, start, len) \ -{ \ - unsigned long bitmap; \ - const unsigned int __bpf = btrfs_blocks_per_folio(fs_info, folio); \ - \ - GET_SUBPAGE_BITMAP(fs_info, folio, name, &bitmap); \ - btrfs_warn(fs_info, \ - "dumping bitmap start=%llu len=%u folio=%llu " #name "_bitmap=%*pbl", \ - start, len, folio_pos(folio), __bpf, &bitmap); \ +DEFINE_GET_SUBPAGE_BITMAP(uptodate); +DEFINE_GET_SUBPAGE_BITMAP(dirty); +DEFINE_GET_SUBPAGE_BITMAP(writeback); + +#define SUBPAGE_DUMP_BITMAP(fs_info, folio, name, start, len) \ +{ \ + const unsigned int __bpf = btrfs_blocks_per_folio(fs_info, folio); \ + \ + if (__bpf <= BITS_PER_LONG) { \ + unsigned long bitmap = get_bitmap_value_##name(fs_info, folio); \ + \ + btrfs_warn(fs_info, \ + "dumping bitmap start=%llu len=%u folio=%llu " #name "_bitmap=%*pbl", \ + start, len, folio_pos(folio), __bpf, &bitmap); \ + } else { \ + btrfs_warn(fs_info, \ + "dumping bitmap start=%llu len=%u folio=%llu " #name "_bitmap=%*pbl", \ + start, len, folio_pos(folio), __bpf, \ + get_bitmap_pointer_##name(fs_info, folio)); \ + } \ } /* @@ -728,7 +651,6 @@ void btrfs_folio_set_lock(const struct btrfs_fs_info *fs_info, { struct btrfs_folio_state *bfs; unsigned long flags; - unsigned int start_bit; unsigned int nbits; int ret; @@ -737,15 +659,8 @@ void btrfs_folio_set_lock(const struct btrfs_fs_info *fs_info, return; bfs = folio_get_private(folio); - start_bit = subpage_calc_start_bit(fs_info, folio, locked, start, len); nbits = len >> fs_info->sectorsize_bits; spin_lock_irqsave(&bfs->lock, flags); - /* Target range should not yet be locked. */ - if (unlikely(!bitmap_test_range_all_zero(bfs->bitmaps, start_bit, nbits))) { - SUBPAGE_DUMP_BITMAP(fs_info, folio, locked, start, len); - ASSERT(bitmap_test_range_all_zero(bfs->bitmaps, start_bit, nbits)); - } - bitmap_set(bfs->bitmaps, start_bit, nbits); ret = atomic_add_return(nbits, &bfs->nr_locked); ASSERT(ret <= btrfs_blocks_per_folio(fs_info, folio)); spin_unlock_irqrestore(&bfs->lock, flags); @@ -778,51 +693,74 @@ void __cold btrfs_subpage_dump_bitmap(const struct btrfs_fs_info *fs_info, { struct btrfs_folio_state *bfs; const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio); - unsigned long uptodate_bitmap; - unsigned long dirty_bitmap; - unsigned long writeback_bitmap; - unsigned long ordered_bitmap; - unsigned long checked_bitmap; - unsigned long locked_bitmap; unsigned long flags; ASSERT(folio_test_private(folio) && folio_get_private(folio)); ASSERT(blocks_per_folio > 1); bfs = folio_get_private(folio); - spin_lock_irqsave(&bfs->lock, flags); - GET_SUBPAGE_BITMAP(fs_info, folio, uptodate, &uptodate_bitmap); - GET_SUBPAGE_BITMAP(fs_info, folio, dirty, &dirty_bitmap); - GET_SUBPAGE_BITMAP(fs_info, folio, writeback, &writeback_bitmap); - GET_SUBPAGE_BITMAP(fs_info, folio, ordered, &ordered_bitmap); - GET_SUBPAGE_BITMAP(fs_info, folio, checked, &checked_bitmap); - GET_SUBPAGE_BITMAP(fs_info, folio, locked, &locked_bitmap); - spin_unlock_irqrestore(&bfs->lock, flags); - dump_page(folio_page(folio, 0), "btrfs folio state dump"); + + if (blocks_per_folio <= BITS_PER_LONG) { + unsigned long uptodate; + unsigned long dirty; + unsigned long writeback; + + spin_lock_irqsave(&bfs->lock, flags); + uptodate = get_bitmap_value_uptodate(fs_info, folio); + dirty = get_bitmap_value_dirty(fs_info, folio); + writeback = get_bitmap_value_writeback(fs_info, folio); + + spin_unlock_irqrestore(&bfs->lock, flags); + + btrfs_warn(fs_info, +"start=%llu len=%u page=%llu, bitmaps uptodate=%*pbl dirty=%*pbl writeback=%*pbl", + start, len, folio_pos(folio), + blocks_per_folio, &uptodate, + blocks_per_folio, &dirty, + blocks_per_folio, &writeback); + return; + } + + spin_lock_irqsave(&bfs->lock, flags); btrfs_warn(fs_info, -"start=%llu len=%u page=%llu, bitmaps uptodate=%*pbl dirty=%*pbl locked=%*pbl writeback=%*pbl ordered=%*pbl checked=%*pbl", +"start=%llu len=%u page=%llu, bitmaps uptodate=%*pbl dirty=%*pbl writeback=%*pbl", start, len, folio_pos(folio), - blocks_per_folio, &uptodate_bitmap, - blocks_per_folio, &dirty_bitmap, - blocks_per_folio, &locked_bitmap, - blocks_per_folio, &writeback_bitmap, - blocks_per_folio, &ordered_bitmap, - blocks_per_folio, &checked_bitmap); + blocks_per_folio, get_bitmap_pointer_uptodate(fs_info, folio), + blocks_per_folio, get_bitmap_pointer_dirty(fs_info, folio), + blocks_per_folio, get_bitmap_pointer_writeback(fs_info, folio)); + spin_unlock_irqrestore(&bfs->lock, flags); } -void btrfs_get_subpage_dirty_bitmap(struct btrfs_fs_info *fs_info, - struct folio *folio, - unsigned long *ret_bitmap) +void btrfs_copy_subpage_dirty_bitmap(struct btrfs_fs_info *fs_info, + struct folio *folio, + unsigned long *dst) { struct btrfs_folio_state *bfs; + const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio); unsigned long flags; + unsigned long value; + + if (blocks_per_folio == 1) { + value = 1; + bitmap_copy(dst, &value, 1); + return; + } ASSERT(folio_test_private(folio) && folio_get_private(folio)); - ASSERT(btrfs_blocks_per_folio(fs_info, folio) > 1); + ASSERT(blocks_per_folio > 1); bfs = folio_get_private(folio); + if (blocks_per_folio <= BITS_PER_LONG) { + spin_lock_irqsave(&bfs->lock, flags); + value = bitmap_read(bfs->bitmaps, btrfs_bitmap_nr_dirty * blocks_per_folio, + blocks_per_folio); + spin_unlock_irqrestore(&bfs->lock, flags); + bitmap_copy(dst, &value, blocks_per_folio); + return; + } spin_lock_irqsave(&bfs->lock, flags); - GET_SUBPAGE_BITMAP(fs_info, folio, dirty, ret_bitmap); + bitmap_copy(dst, get_bitmap_pointer_dirty(fs_info, folio), + blocks_per_folio); spin_unlock_irqrestore(&bfs->lock, flags); } diff --git a/fs/btrfs/subpage.h b/fs/btrfs/subpage.h index d81a0ade559f..c6d7394e6418 100644 --- a/fs/btrfs/subpage.h +++ b/fs/btrfs/subpage.h @@ -14,15 +14,15 @@ struct folio; /* * Extra info for subpage bitmap. * - * For subpage we pack all uptodate/dirty/writeback/ordered bitmaps into + * For subpage we pack all uptodate/dirty/writeback bitmaps into * one larger bitmap. * * This structure records how they are organized in the bitmap: * - * /- uptodate /- dirty /- ordered + * /- uptodate /- dirty /- writeback * | | | * v v v - * |u|u|u|u|........|u|u|d|d|.......|d|d|o|o|.......|o|o| + * |u|u|u|u|........|u|u|d|d|.......|d|d|w|w|.......|w|w| * |< sectors_per_page >| * * Unlike regular macro-like enums, here we do not go upper-case names, as @@ -40,23 +40,6 @@ enum { */ btrfs_bitmap_nr_writeback, - /* - * The ordered and checked flags are for COW fixup, already marked - * deprecated, and will be removed eventually. - */ - btrfs_bitmap_nr_ordered, - btrfs_bitmap_nr_checked, - - /* - * The locked bit is for async delalloc range (compression), currently - * async extent is queued with the range locked, until the compression - * is done. - * So an async extent can unlock the range at any random timing. - * - * This will need a rework on the async extent lifespan (mark writeback - * and do compression) before deprecating this flag. - */ - btrfs_bitmap_nr_locked, btrfs_bitmap_nr_max }; @@ -133,7 +116,7 @@ void btrfs_folio_end_lock(const struct btrfs_fs_info *fs_info, void btrfs_folio_set_lock(const struct btrfs_fs_info *fs_info, struct folio *folio, u64 start, u32 len); void btrfs_folio_end_lock_bitmap(const struct btrfs_fs_info *fs_info, - struct folio *folio, unsigned long bitmap); + struct folio *folio, unsigned long *bitmap); /* * Template for subpage related operations. * @@ -181,8 +164,6 @@ bool btrfs_meta_folio_test_##name(struct folio *folio, const struct extent_buffe DECLARE_BTRFS_SUBPAGE_OPS(uptodate); DECLARE_BTRFS_SUBPAGE_OPS(dirty); DECLARE_BTRFS_SUBPAGE_OPS(writeback); -DECLARE_BTRFS_SUBPAGE_OPS(ordered); -DECLARE_BTRFS_SUBPAGE_OPS(checked); /* * Helper for error cleanup, where a folio will have its dirty flag cleared, @@ -203,9 +184,9 @@ bool btrfs_subpage_clear_and_test_dirty(const struct btrfs_fs_info *fs_info, void btrfs_folio_assert_not_dirty(const struct btrfs_fs_info *fs_info, struct folio *folio, u64 start, u32 len); bool btrfs_meta_folio_clear_and_test_dirty(struct folio *folio, const struct extent_buffer *eb); -void btrfs_get_subpage_dirty_bitmap(struct btrfs_fs_info *fs_info, - struct folio *folio, - unsigned long *ret_bitmap); +void btrfs_copy_subpage_dirty_bitmap(struct btrfs_fs_info *fs_info, + struct folio *folio, + unsigned long *dst); void __cold btrfs_subpage_dump_bitmap(const struct btrfs_fs_info *fs_info, struct folio *folio, u64 start, u32 len); diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c index 636154861d7c..a7d804219bec 100644 --- a/fs/btrfs/super.c +++ b/fs/btrfs/super.c @@ -60,6 +60,7 @@ #include "verity.h" #include "super.h" #include "extent-tree.h" +#include "tree-log.h" #define CREATE_TRACE_POINTS #include @@ -1633,8 +1634,7 @@ static inline int btrfs_calc_avail_data_space(struct btrfs_fs_info *fs_info, } } - devices_info = kmalloc_array(nr_devices, sizeof(*devices_info), - GFP_KERNEL); + devices_info = kmalloc_objs(*devices_info, nr_devices); if (!devices_info) return -ENOMEM; @@ -1732,15 +1732,17 @@ static int btrfs_statfs(struct dentry *dentry, struct kstatfs *buf) u64 total_free_data = 0; u64 total_free_meta = 0; u32 bits = fs_info->sectorsize_bits; - __be32 *fsid = (__be32 *)fs_info->fs_devices->fsid; + __be32 *fsid; unsigned factor = 1; struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv; int ret; u64 thresh = 0; - int mixed = 0; + bool mixed = false; + __kernel_fsid_t f_fsid; list_for_each_entry(found, &fs_info->space_info, list) { - if (found->flags & BTRFS_BLOCK_GROUP_DATA) { + if (found->flags & BTRFS_BLOCK_GROUP_DATA && + found->subgroup_id != BTRFS_SUB_GROUP_DATA_RELOC) { int i; total_free_data += found->disk_total - found->disk_used; @@ -1759,7 +1761,7 @@ static int btrfs_statfs(struct dentry *dentry, struct kstatfs *buf) */ if (!mixed && found->flags & BTRFS_BLOCK_GROUP_METADATA) { if (found->flags & BTRFS_BLOCK_GROUP_DATA) - mixed = 1; + mixed = true; else total_free_meta += found->disk_total - found->disk_used; @@ -1818,14 +1820,38 @@ static int btrfs_statfs(struct dentry *dentry, struct kstatfs *buf) buf->f_bsize = fs_info->sectorsize; buf->f_namelen = BTRFS_NAME_LEN; - /* We treat it as constant endianness (it doesn't matter _which_) - because we want the fsid to come out the same whether mounted - on a big-endian or little-endian host */ - buf->f_fsid.val[0] = be32_to_cpu(fsid[0]) ^ be32_to_cpu(fsid[2]); - buf->f_fsid.val[1] = be32_to_cpu(fsid[1]) ^ be32_to_cpu(fsid[3]); + /* + * fs_devices->fsid is dynamically generated when temp_fsid is active + * to support cloned filesystems. Use the original on-disk fsid instead, + * as it remains consistent across mount cycles. + */ + if (fs_info->fs_devices->temp_fsid) + fsid = (__be32 *)fs_info->super_copy->fsid; + else + fsid = (__be32 *)fs_info->fs_devices->fsid; + + /* + * We treat it as constant endianness (it doesn't matter _which_) + * because we want the fsid to come out the same whether mounted + * on a big-endian or little-endian host. + */ + f_fsid.val[0] = be32_to_cpu(fsid[0]) ^ be32_to_cpu(fsid[2]); + f_fsid.val[1] = be32_to_cpu(fsid[1]) ^ be32_to_cpu(fsid[3]); + /* Mask in the root object ID too, to disambiguate subvols */ - buf->f_fsid.val[0] ^= btrfs_root_id(BTRFS_I(d_inode(dentry))->root) >> 32; - buf->f_fsid.val[1] ^= btrfs_root_id(BTRFS_I(d_inode(dentry))->root); + f_fsid.val[0] ^= btrfs_root_id(BTRFS_I(d_inode(dentry))->root) >> 32; + f_fsid.val[1] ^= btrfs_root_id(BTRFS_I(d_inode(dentry))->root); + + /* Hash dev_t to avoid f_fsid collision with cloned filesystems. */ + if (fs_info->fs_devices->total_devices == 1) { + __kernel_fsid_t dev_fsid = + u64_to_fsid(huge_encode_dev(fs_info->fs_devices->latest_dev->bdev->bd_dev)); + + f_fsid.val[0] ^= dev_fsid.val[1]; + f_fsid.val[1] ^= dev_fsid.val[0]; + } + + memcpy(&buf->f_fsid, &f_fsid, sizeof(f_fsid)); return 0; } @@ -1873,6 +1899,7 @@ static int btrfs_get_tree_super(struct fs_context *fc) fs_info->fs_devices = fs_devices; mutex_unlock(&uuid_mutex); + fc->sb_flags |= SB_NOSEC; sb = sget_fc(fc, btrfs_fc_test_super, set_anon_super_fc); if (IS_ERR(sb)) { @@ -2405,7 +2432,7 @@ static int btrfs_show_devname(struct seq_file *m, struct dentry *root) static long btrfs_nr_cached_objects(struct super_block *sb, struct shrink_control *sc) { struct btrfs_fs_info *fs_info = btrfs_sb(sb); - const s64 nr = percpu_counter_sum_positive(&fs_info->evictable_extent_maps); + const s64 nr = percpu_counter_read_positive(&fs_info->evictable_extent_maps); trace_btrfs_extent_map_shrinker_count(fs_info, nr); @@ -2604,6 +2631,9 @@ static const struct init_sequence mod_init_seq[] = { }, { .init_func = btrfs_init_compress, .exit_func = btrfs_exit_compress, + }, { + .init_func = btrfs_init_block_group, + .exit_func = btrfs_exit_block_group, }, { .init_func = btrfs_init_cachep, .exit_func = btrfs_destroy_cachep, diff --git a/fs/btrfs/tests/btrfs-tests.c b/fs/btrfs/tests/btrfs-tests.c index 19c127ac6d10..6287d940323d 100644 --- a/fs/btrfs/tests/btrfs-tests.c +++ b/fs/btrfs/tests/btrfs-tests.c @@ -145,6 +145,7 @@ struct btrfs_fs_info *btrfs_alloc_dummy_fs_info(u32 nodesize, u32 sectorsize) fs_info->csum_size = 4; fs_info->csums_per_leaf = BTRFS_MAX_ITEM_SIZE(fs_info) / fs_info->csum_size; + fs_info->use_bitmap = btrfs_use_bitmap; set_bit(BTRFS_FS_STATE_DUMMY_FS_INFO, &fs_info->fs_state); test_mnt->mnt_sb->s_fs_info = fs_info; diff --git a/fs/btrfs/tests/free-space-tests.c b/fs/btrfs/tests/free-space-tests.c index ebf68fcd2149..0425b3b68716 100644 --- a/fs/btrfs/tests/free-space-tests.c +++ b/fs/btrfs/tests/free-space-tests.c @@ -398,10 +398,8 @@ test_steal_space_from_bitmap_to_extent(struct btrfs_block_group *cache, int ret; u64 offset; u64 max_extent_size; - const struct btrfs_free_space_op test_free_space_ops = { - .use_bitmap = test_use_bitmap, - }; - const struct btrfs_free_space_op *orig_free_space_ops; + bool (*orig_use_bitmap)(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *info); test_msg("running space stealing from bitmap to extent tests"); @@ -423,8 +421,8 @@ test_steal_space_from_bitmap_to_extent(struct btrfs_block_group *cache, * that forces use of bitmaps as soon as we have at least 1 * extent entry. */ - orig_free_space_ops = cache->free_space_ctl->op; - cache->free_space_ctl->op = &test_free_space_ops; + orig_use_bitmap = cache->fs_info->use_bitmap; + cache->fs_info->use_bitmap = test_use_bitmap; /* * Extent entry covering free space range [128Mb - 256Kb, 128Mb - 128Kb[ @@ -818,7 +816,7 @@ test_steal_space_from_bitmap_to_extent(struct btrfs_block_group *cache, if (ret) return ret; - cache->free_space_ctl->op = orig_free_space_ops; + cache->fs_info->use_bitmap = orig_use_bitmap; btrfs_remove_free_space_cache(cache); return 0; @@ -832,10 +830,8 @@ static bool bytes_index_use_bitmap(struct btrfs_free_space_ctl *ctl, static int test_bytes_index(struct btrfs_block_group *cache, u32 sectorsize) { - const struct btrfs_free_space_op test_free_space_ops = { - .use_bitmap = bytes_index_use_bitmap, - }; - const struct btrfs_free_space_op *orig_free_space_ops; + bool (*orig_use_bitmap)(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *info); struct btrfs_free_space_ctl *ctl = cache->free_space_ctl; struct btrfs_free_space *entry; struct rb_node *node; @@ -892,8 +888,8 @@ static int test_bytes_index(struct btrfs_block_group *cache, u32 sectorsize) /* Now validate bitmaps with different ->max_extent_size. */ btrfs_remove_free_space_cache(cache); - orig_free_space_ops = cache->free_space_ctl->op; - cache->free_space_ctl->op = &test_free_space_ops; + orig_use_bitmap = cache->fs_info->use_bitmap; + cache->fs_info->use_bitmap = bytes_index_use_bitmap; ret = test_add_free_space_entry(cache, 0, sectorsize, 1); if (ret) { @@ -997,7 +993,7 @@ static int test_bytes_index(struct btrfs_block_group *cache, u32 sectorsize) return -EINVAL; } - cache->free_space_ctl->op = orig_free_space_ops; + cache->fs_info->use_bitmap = orig_use_bitmap; btrfs_remove_free_space_cache(cache); return 0; } diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index 194f581b36f3..8f9419728100 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -394,6 +394,7 @@ loop: cur_trans->transid = fs_info->generation; fs_info->running_transaction = cur_trans; cur_trans->aborted = 0; + trace_btrfs_transaction_start(cur_trans); spin_unlock(&fs_info->trans_lock); return 0; @@ -630,7 +631,7 @@ start_transaction(struct btrfs_root *root, unsigned int num_items, * the appropriate flushing if need be. */ if (num_items && root != fs_info->chunk_root) { - qgroup_reserved = num_items * fs_info->nodesize; + qgroup_reserved = (num_items << fs_info->nodesize_bits); /* * Use prealloc for now, as there might be a currently running * transaction that could free this reserved space prematurely @@ -2114,7 +2115,7 @@ static void cleanup_transaction(struct btrfs_trans_handle *trans, int err) btrfs_put_transaction(cur_trans); btrfs_put_transaction(cur_trans); - trace_btrfs_transaction_commit(fs_info); + trace_btrfs_transaction_commit(trans); if (current->journal_info == trans) current->journal_info = NULL; @@ -2266,7 +2267,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) btrfs_create_pending_block_groups(trans); if (!test_bit(BTRFS_TRANS_DIRTY_BG_RUN, &cur_trans->flags)) { - int run_it = 0; + bool run_it = false; /* this mutex is also taken before trying to set * block groups readonly. We need to make sure @@ -2284,7 +2285,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) mutex_lock(&fs_info->ro_block_group_mutex); if (!test_and_set_bit(BTRFS_TRANS_DIRTY_BG_RUN, &cur_trans->flags)) - run_it = 1; + run_it = true; mutex_unlock(&fs_info->ro_block_group_mutex); if (run_it) { @@ -2320,6 +2321,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) } cur_trans->state = TRANS_STATE_COMMIT_PREP; + trace_btrfs_transaction_commit(trans); wake_up(&fs_info->transaction_blocked_wait); btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_COMMIT_PREP); @@ -2358,6 +2360,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) } cur_trans->state = TRANS_STATE_COMMIT_START; + trace_btrfs_transaction_commit(trans); wake_up(&fs_info->transaction_blocked_wait); spin_unlock(&fs_info->trans_lock); @@ -2413,6 +2416,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) spin_lock(&fs_info->trans_lock); add_pending_snapshot(trans); cur_trans->state = TRANS_STATE_COMMIT_DOING; + trace_btrfs_transaction_commit(trans); spin_unlock(&fs_info->trans_lock); /* @@ -2561,6 +2565,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) spin_lock(&fs_info->trans_lock); cur_trans->state = TRANS_STATE_UNBLOCKED; + trace_btrfs_transaction_commit(trans); fs_info->running_transaction = NULL; spin_unlock(&fs_info->trans_lock); mutex_unlock(&fs_info->reloc_mutex); @@ -2603,6 +2608,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) * which can change it. */ cur_trans->state = TRANS_STATE_SUPER_COMMITTED; + trace_btrfs_transaction_commit(trans); wake_up(&cur_trans->commit_wait); btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_SUPER_COMMITTED); @@ -2619,6 +2625,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) * which can change it. */ cur_trans->state = TRANS_STATE_COMPLETED; + trace_btrfs_transaction_commit(trans); wake_up(&cur_trans->commit_wait); btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_COMPLETED); @@ -2632,8 +2639,6 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) if (trans->type & __TRANS_FREEZABLE) sb_end_intwrite(fs_info->sb); - trace_btrfs_transaction_commit(fs_info); - btrfs_scrub_continue(fs_info); if (current->journal_info == trans) @@ -2722,17 +2727,33 @@ int btrfs_clean_one_deleted_snapshot(struct btrfs_fs_info *fs_info) * * We'll complete the cleanup in btrfs_end_transaction and * btrfs_commit_transaction. + * + * Note: the parameter @error encodes whether the transactin abort was first hit + * (setting the FS_ERROR state bit in btrfs_abort_transaction()) + * - positive number - first hit + * - negative number - abort after it was already done */ void __cold __btrfs_abort_transaction(struct btrfs_trans_handle *trans, const char *function, - unsigned int line, int error, bool first_hit) + unsigned int line, int error) { struct btrfs_fs_info *fs_info = trans->fs_info; + bool first_hit = false; + + if (error > 0) { + error = -error; + first_hit = true; + } WRITE_ONCE(trans->aborted, error); WRITE_ONCE(trans->transaction->aborted, error); - if (first_hit && error == -ENOSPC) - btrfs_dump_space_info_for_trans_abort(fs_info); + trace_btrfs_transaction_abort(trans); + if (first_hit) { + btrfs_err(fs_info, "Transaction %llu aborted (error %d)", + trans->transid, error); + if (error == -ENOSPC) + btrfs_dump_space_info_for_trans_abort(fs_info); + } /* Wake up anybody who may be waiting on this transaction */ wake_up(&fs_info->transaction_wait); wake_up(&fs_info->transaction_blocked_wait); diff --git a/fs/btrfs/transaction.h b/fs/btrfs/transaction.h index 7d70fe486758..5e4b1106fd90 100644 --- a/fs/btrfs/transaction.h +++ b/fs/btrfs/transaction.h @@ -243,29 +243,47 @@ static inline bool btrfs_abort_should_print_stack(int error) } /* - * Call btrfs_abort_transaction as early as possible when an error condition is - * detected, that way the exact stack trace is reported for some errors. + * Compile-time and run-time verification of error passed to transaction abort. + * Direct constants will be caught at compile time, errors read from variables + * can be caught only at run-time and will warn under debugging config. + * + * How verification works: + * - accepted builtin constants are all -EIO and such + * - for compile-time check, invalid condition produces a negative-sized array + * type, valid zero-sized + * - when a variable is passed as error the first check is a no-op + * - with enabled debugging, the second array type size is constructed from the + * real variable value, valid condition produces array of size 1 + * - sizeof(type) does not generate any code + */ +#define VERIFY_NEGATIVE_ERROR(error) \ +do { \ + (void)sizeof(char[-!(__builtin_constant_p(error) ? (error) < 0 : 1)]); \ + if (IS_ENABLED(CONFIG_BTRFS_DEBUG)) { \ + if (sizeof(char[(error) < 0]) != 1) \ + DEBUG_WARN("error >= 0 passed to btrfs_abort_transaction()"); \ + } \ +} while(0) + +/* + * Call btrfs_abort_transaction() as early as possible when an error condition + * is detected, that way the exact stack trace is reported for some errors. + * + * Error number must be negative as it encodes wheather it's the first abort. */ #define btrfs_abort_transaction(trans, error) \ do { \ - bool __first = false; \ + int __error = (error); \ + \ + VERIFY_NEGATIVE_ERROR(error); \ /* Report first abort since mount */ \ if (!test_and_set_bit(BTRFS_FS_STATE_TRANS_ABORTED, \ &((trans)->fs_info->fs_state))) { \ - __first = true; \ - if (WARN(btrfs_abort_should_print_stack(error), \ - KERN_ERR \ - "BTRFS: Transaction aborted (error %d)\n", \ - (error))) { \ - /* Stack trace printed. */ \ - } else { \ - btrfs_err((trans)->fs_info, \ - "Transaction aborted (error %d)", \ - (error)); \ - } \ + WARN_ON(btrfs_abort_should_print_stack(__error)); \ + __error = -__error; \ } \ __btrfs_abort_transaction((trans), __func__, \ - __LINE__, (error), __first); \ + __LINE__, __error); \ } while (0) int btrfs_end_transaction(struct btrfs_trans_handle *trans); @@ -303,7 +321,7 @@ void btrfs_add_dropped_root(struct btrfs_trans_handle *trans, void btrfs_trans_release_chunk_metadata(struct btrfs_trans_handle *trans); void __cold __btrfs_abort_transaction(struct btrfs_trans_handle *trans, const char *function, - unsigned int line, int error, bool first_hit); + unsigned int line, int error); int __init btrfs_transaction_init(void); void __cold btrfs_transaction_exit(void); diff --git a/fs/btrfs/tree-checker.c b/fs/btrfs/tree-checker.c index 1f15d0793a9c..cb3e676a81cc 100644 --- a/fs/btrfs/tree-checker.c +++ b/fs/btrfs/tree-checker.c @@ -296,6 +296,33 @@ static int check_extent_data_item(struct extent_buffer *leaf, return 0; } + /* + * For the data reloc tree, file extent items are written by + * relocation's own paths. The data reloc inode is created with + * BTRFS_INODE_NOCOMPRESS, so insert_ordered_extent_file_extent() + * always leaves the compression field at 0. Encryption and + * other_encoding are reserved-and-zero in btrfs. A non-zero value + * for any of these means the leaf decoded from disk does not match + * what the kernel wrote, i.e. on-disk corruption. + * + * The file_extent_item's offset field is NOT a universal invariant + * here: partial-PREALLOC writebacks legitimately produce REG items + * with non-zero offset at non-boundary keys. The offset check is + * performed at the call site in get_new_location(), which only + * inspects cluster-boundary keys where offset is always 0. + */ + if (unlikely(btrfs_header_owner(leaf) == BTRFS_DATA_RELOC_TREE_OBJECTID && + (btrfs_file_extent_compression(leaf, fi) || + btrfs_file_extent_encryption(leaf, fi) || + btrfs_file_extent_other_encoding(leaf, fi)))) { + file_extent_err(leaf, slot, +"invalid encoding fields for data reloc tree, compression=%u encryption=%u other_encoding=%u", + btrfs_file_extent_compression(leaf, fi), + btrfs_file_extent_encryption(leaf, fi), + btrfs_file_extent_other_encoding(leaf, fi)); + return -EUCLEAN; + } + /* Regular or preallocated extent has fixed item size */ if (unlikely(item_size != sizeof(*fi))) { file_extent_err(leaf, slot, @@ -1371,6 +1398,37 @@ static int check_root_item(struct extent_buffer *leaf, struct btrfs_key *key, return 0; } +static int check_root_ref(struct extent_buffer *leaf, struct btrfs_key *key, int slot) +{ + struct btrfs_root_ref *rref; + u32 item_size = btrfs_item_size(leaf, slot); + u32 name_len; + + if (unlikely(item_size <= sizeof(*rref))) { + generic_err(leaf, slot, + "invalid root ref item size for key type %u, have %u expect > %zu", + key->type, item_size, sizeof(*rref)); + return -EUCLEAN; + } + + rref = btrfs_item_ptr(leaf, slot, struct btrfs_root_ref); + name_len = btrfs_root_ref_name_len(leaf, rref); + if (unlikely(name_len > BTRFS_NAME_LEN)) { + generic_err(leaf, slot, + "root ref name too long for key type %u, have %u max %u", + key->type, name_len, BTRFS_NAME_LEN); + return -EUCLEAN; + } + if (unlikely(item_size != sizeof(*rref) + name_len)) { + generic_err(leaf, slot, + "invalid root ref item size for key type %u, have %u expect %zu", + key->type, item_size, sizeof(*rref) + name_len); + return -EUCLEAN; + } + + return 0; +} + __printf(3,4) __cold static void extent_err(const struct extent_buffer *eb, int slot, @@ -2071,6 +2129,7 @@ static int check_free_space_info(struct extent_buffer *leaf, struct btrfs_key *k struct btrfs_fs_info *fs_info = leaf->fs_info; struct btrfs_free_space_info *fsi; const u32 blocksize = fs_info->sectorsize; + u64 end; u32 flags; if (unlikely(!IS_ALIGNED(key->objectid, blocksize))) { @@ -2085,6 +2144,12 @@ static int check_free_space_info(struct extent_buffer *leaf, struct btrfs_key *k blocksize, BTRFS_KEY_FMT_VALUE(key)); return -EUCLEAN; } + if (unlikely(check_add_overflow(key->objectid, key->offset, &end))) { + generic_err(leaf, slot, + "free space info key overflows, has " BTRFS_KEY_FMT, + BTRFS_KEY_FMT_VALUE(key)); + return -EUCLEAN; + } if (unlikely(btrfs_item_size(leaf, slot) != sizeof(struct btrfs_free_space_info))) { generic_err(leaf, slot, @@ -2112,23 +2177,98 @@ static int check_free_space_info(struct extent_buffer *leaf, struct btrfs_key *k return 0; } -static int check_free_space_extent(struct extent_buffer *leaf, struct btrfs_key *key, int slot) +static int check_free_space_common_key(struct extent_buffer *leaf, struct btrfs_key *key, int slot, + struct btrfs_key *prev_key) { struct btrfs_fs_info *fs_info = leaf->fs_info; const u32 blocksize = fs_info->sectorsize; + const char *type_str = (key->type == BTRFS_FREE_SPACE_EXTENT_KEY) ? "extent" : "bitmap"; + u64 end; if (unlikely(!IS_ALIGNED(key->objectid, blocksize))) { generic_err(leaf, slot, - "free space extent key objectid is not aligned to %u, has " BTRFS_KEY_FMT, - blocksize, BTRFS_KEY_FMT_VALUE(key)); + "free space %s key objectid is not aligned to %u, has " BTRFS_KEY_FMT, + type_str, blocksize, BTRFS_KEY_FMT_VALUE(key)); return -EUCLEAN; } if (unlikely(!IS_ALIGNED(key->offset, blocksize))) { generic_err(leaf, slot, - "free space extent key offset is not aligned to %u, has " BTRFS_KEY_FMT, - blocksize, BTRFS_KEY_FMT_VALUE(key)); + "free space %s key offset is not aligned to %u, has " BTRFS_KEY_FMT, + type_str, blocksize, BTRFS_KEY_FMT_VALUE(key)); return -EUCLEAN; } + if (unlikely(key->offset == 0)) { + generic_err(leaf, slot, "free space %s length is 0", type_str); + return -EUCLEAN; + } + if (unlikely(check_add_overflow(key->objectid, key->offset, &end))) { + generic_err(leaf, slot, + "free space %s end overflow, have objectid %llu offset %llu", + type_str, key->objectid, key->offset); + return -EUCLEAN; + } + if (slot == 0) + return 0; + + /* + * Make sure the current key is inside the block group, and matching + * the expected info type. + */ + if (prev_key->type == BTRFS_FREE_SPACE_INFO_KEY) { + struct btrfs_free_space_info *fsi; + u32 info_flags; + + if (unlikely(key->objectid < prev_key->objectid || + key->objectid + key->offset > prev_key->objectid + prev_key->offset)) { + generic_err(leaf, slot, +"free space %s is not inside the space info, prev key " BTRFS_KEY_FMT " current key " BTRFS_KEY_FMT, + type_str, BTRFS_KEY_FMT_VALUE(prev_key), + BTRFS_KEY_FMT_VALUE(key)); + return -EUCLEAN; + } + fsi = btrfs_item_ptr(leaf, slot - 1, struct btrfs_free_space_info); + info_flags = btrfs_free_space_flags(leaf, fsi); + if (unlikely((info_flags == BTRFS_FREE_SPACE_USING_BITMAPS && + key->type == BTRFS_FREE_SPACE_EXTENT_KEY) || + (info_flags != BTRFS_FREE_SPACE_USING_BITMAPS && + key->type == BTRFS_FREE_SPACE_BITMAP_KEY))) { + generic_err(leaf, slot, +"free space %s key type is not matching the type of space info, key type %u space info flags %u", + type_str, key->type, info_flags); + return -EUCLEAN; + } + return 0; + } + /* + * Previous key should be either FREE_SPACE_EXTENT or FREE_SPACE_BITMAP. + * Inside the same block group the key type should match each other, and + * no overlaps. + */ + if (unlikely(key->type != prev_key->type)) { + generic_err(leaf, slot, +"free space %s key type is not matching the type of previous key, key type %u prev key type %u", + type_str, key->type, prev_key->type); + return -EUCLEAN; + } + if (unlikely(prev_key->objectid + prev_key->offset > key->objectid)) { + generic_err(leaf, slot, +"free space %s key overlaps previous key, prev key " BTRFS_KEY_FMT " current key " BTRFS_KEY_FMT, + type_str, BTRFS_KEY_FMT_VALUE(prev_key), + BTRFS_KEY_FMT_VALUE(key)); + return -EUCLEAN; + } + return 0; +} + +static int check_free_space_extent(struct extent_buffer *leaf, struct btrfs_key *key, int slot, + struct btrfs_key *prev_key) +{ + int ret; + + ret = check_free_space_common_key(leaf, key, slot, prev_key); + if (unlikely(ret < 0)) + return ret; + if (unlikely(btrfs_item_size(leaf, slot) != 0)) { generic_err(leaf, slot, "invalid item size for free space info, has %u expect 0", @@ -2139,28 +2279,17 @@ static int check_free_space_extent(struct extent_buffer *leaf, struct btrfs_key } static int check_free_space_bitmap(struct extent_buffer *leaf, - struct btrfs_key *key, int slot) + struct btrfs_key *key, int slot, + struct btrfs_key *prev_key) { struct btrfs_fs_info *fs_info = leaf->fs_info; - const u32 blocksize = fs_info->sectorsize; u32 expected_item_size; + int ret; + + ret = check_free_space_common_key(leaf, key, slot, prev_key); + if (unlikely(ret < 0)) + return ret; - if (unlikely(!IS_ALIGNED(key->objectid, blocksize))) { - generic_err(leaf, slot, - "free space bitmap key objectid is not aligned to %u, has " BTRFS_KEY_FMT, - blocksize, BTRFS_KEY_FMT_VALUE(key)); - return -EUCLEAN; - } - if (unlikely(!IS_ALIGNED(key->offset, blocksize))) { - generic_err(leaf, slot, - "free space bitmap key offset is not aligned to %u, has " BTRFS_KEY_FMT, - blocksize, BTRFS_KEY_FMT_VALUE(key)); - return -EUCLEAN; - } - if (unlikely(key->offset == 0)) { - generic_err(leaf, slot, "free space bitmap length is 0"); - return -EUCLEAN; - } /* * The item must hold exactly the right number of bitmap bytes for the * range described by key->offset. A mismatch means the item was @@ -2226,6 +2355,10 @@ static enum btrfs_tree_block_status check_leaf_item(struct extent_buffer *leaf, case BTRFS_ROOT_ITEM_KEY: ret = check_root_item(leaf, key, slot); break; + case BTRFS_ROOT_REF_KEY: + case BTRFS_ROOT_BACKREF_KEY: + ret = check_root_ref(leaf, key, slot); + break; case BTRFS_EXTENT_ITEM_KEY: case BTRFS_METADATA_ITEM_KEY: ret = check_extent_item(leaf, key, slot, prev_key); @@ -2245,10 +2378,10 @@ static enum btrfs_tree_block_status check_leaf_item(struct extent_buffer *leaf, ret = check_free_space_info(leaf, key, slot); break; case BTRFS_FREE_SPACE_EXTENT_KEY: - ret = check_free_space_extent(leaf, key, slot); + ret = check_free_space_extent(leaf, key, slot, prev_key); break; case BTRFS_FREE_SPACE_BITMAP_KEY: - ret = check_free_space_bitmap(leaf, key, slot); + ret = check_free_space_bitmap(leaf, key, slot, prev_key); break; case BTRFS_IDENTITY_REMAP_KEY: case BTRFS_REMAP_KEY: diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index 9123adafa0d1..875e4ddc68ea 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -33,17 +33,6 @@ #define MAX_CONFLICT_INODES 10 -/* magic values for the inode_only field in btrfs_log_inode: - * - * LOG_INODE_ALL means to log everything - * LOG_INODE_EXISTS means to log just enough to recreate the inode - * during log replay - */ -enum { - LOG_INODE_ALL, - LOG_INODE_EXISTS, -}; - /* * directory trouble cases * @@ -227,7 +216,7 @@ static void do_abort_log_replay(struct walk_control *wc, const char *function, static int btrfs_log_inode(struct btrfs_trans_handle *trans, struct btrfs_inode *inode, - int inode_only, + enum btrfs_log_mode log_mode, struct btrfs_log_ctx *ctx); static int link_to_fixup_dir(struct walk_control *wc, u64 objectid); static noinline int replay_dir_deletes(struct walk_control *wc, @@ -502,7 +491,7 @@ static int overwrite_item(struct walk_control *wc) * the leaf before writing into the log tree. See the comments at * copy_items() for more details. */ - ASSERT(btrfs_root_id(root) != BTRFS_TREE_LOG_OBJECTID, "root_id=%llu", btrfs_root_id(root)); + ASSERT(btrfs_root_id(root) != BTRFS_TREE_LOG_OBJECTID); item_size = btrfs_item_size(wc->log_leaf, wc->log_slot); src_ptr = btrfs_item_ptr_offset(wc->log_leaf, wc->log_slot); @@ -3333,8 +3322,10 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, u64 log_root_level; mutex_lock(&root->log_mutex); + trace_btrfs_sync_log_enter(trans, root, ctx); log_transid = ctx->log_transid; if (root->log_transid_committed >= log_transid) { + trace_btrfs_sync_log_exit(trans, root, ctx, ctx->log_ret); mutex_unlock(&root->log_mutex); return ctx->log_ret; } @@ -3342,6 +3333,7 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, index1 = log_transid % 2; if (atomic_read(&root->log_commit[index1])) { wait_log_commit(root, log_transid); + trace_btrfs_sync_log_exit(trans, root, ctx, ctx->log_ret); mutex_unlock(&root->log_mutex); return ctx->log_ret; } @@ -3370,6 +3362,7 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, /* bail out if we need to do a full commit */ if (btrfs_need_log_full_commit(trans)) { ret = BTRFS_LOG_FORCE_COMMIT; + trace_btrfs_sync_log_exit(trans, root, ctx, ret); mutex_unlock(&root->log_mutex); goto out; } @@ -3396,6 +3389,7 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, if (ret == -EAGAIN && btrfs_is_zoned(fs_info)) ret = 0; if (ret) { + trace_btrfs_sync_log_exit(trans, root, ctx, ret); blk_finish_plug(&plug); btrfs_set_log_full_commit(trans); mutex_unlock(&root->log_mutex); @@ -3433,6 +3427,7 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, if (!log_root_tree->node) { ret = btrfs_alloc_log_tree_node(trans, log_root_tree); if (ret) { + trace_btrfs_sync_log_exit(trans, root, ctx, ret); mutex_unlock(&fs_info->tree_root->log_mutex); blk_finish_plug(&plug); goto out; @@ -3456,6 +3451,7 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, */ ret = update_log_root(trans, log, &new_root_item); if (ret) { + trace_btrfs_sync_log_exit(trans, root, ctx, ret); list_del_init(&root_log_ctx.list); blk_finish_plug(&plug); btrfs_set_log_full_commit(trans); @@ -3473,6 +3469,7 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, list_del_init(&root_log_ctx.list); mutex_unlock(&log_root_tree->log_mutex); ret = root_log_ctx.log_ret; + trace_btrfs_sync_log_exit(trans, root, ctx, ret); goto out; } @@ -3484,6 +3481,7 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, mutex_unlock(&log_root_tree->log_mutex); if (!ret) ret = root_log_ctx.log_ret; + trace_btrfs_sync_log_exit(trans, root, ctx, ret); goto out; } ASSERT(root_log_ctx.log_transid == log_root_tree->log_transid, @@ -3505,6 +3503,7 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, btrfs_wait_tree_log_extents(log, mark); mutex_unlock(&log_root_tree->log_mutex); ret = BTRFS_LOG_FORCE_COMMIT; + trace_btrfs_sync_log_exit(trans, root, ctx, ret); goto out_wake_log_root; } @@ -3518,11 +3517,13 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, * deadlock. Bail out to the full commit instead. */ if (ret == -EAGAIN && btrfs_is_zoned(fs_info)) { + trace_btrfs_sync_log_exit(trans, root, ctx, ret); btrfs_set_log_full_commit(trans); btrfs_wait_tree_log_extents(log, mark); mutex_unlock(&log_root_tree->log_mutex); goto out_wake_log_root; } else if (ret) { + trace_btrfs_sync_log_exit(trans, root, ctx, ret); btrfs_set_log_full_commit(trans); mutex_unlock(&log_root_tree->log_mutex); goto out_wake_log_root; @@ -3532,6 +3533,7 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, ret = btrfs_wait_tree_log_extents(log_root_tree, EXTENT_DIRTY_LOG1 | EXTENT_DIRTY_LOG2); if (ret) { + trace_btrfs_sync_log_exit(trans, root, ctx, ret); btrfs_set_log_full_commit(trans); mutex_unlock(&log_root_tree->log_mutex); goto out_wake_log_root; @@ -3568,6 +3570,7 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, */ if (unlikely(BTRFS_FS_ERROR(fs_info))) { ret = -EIO; + trace_btrfs_sync_log_exit(trans, root, ctx, ret); btrfs_set_log_full_commit(trans); btrfs_abort_transaction(trans, ret); mutex_unlock(&fs_info->tree_log_mutex); @@ -3579,6 +3582,7 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, ret = write_all_supers(trans); mutex_unlock(&fs_info->tree_log_mutex); if (unlikely(ret)) { + trace_btrfs_sync_log_exit(trans, root, ctx, ret); btrfs_set_log_full_commit(trans); btrfs_abort_transaction(trans, ret); goto out_wake_log_root; @@ -4771,7 +4775,7 @@ static noinline int copy_items(struct btrfs_trans_handle *trans, struct btrfs_inode *inode, struct btrfs_path *dst_path, struct btrfs_path *src_path, - int start_slot, int nr, int inode_only, + int start_slot, int nr, enum btrfs_log_mode log_mode, u64 logged_isize, struct btrfs_log_ctx *ctx) { struct btrfs_root *log = inode->root->log_root; @@ -4985,7 +4989,7 @@ copy_item: inode_item = btrfs_item_ptr(dst_path->nodes[0], dst_slot, struct btrfs_inode_item); fill_inode_item(trans, dst_path->nodes[0], inode_item, - inode, inode_only == LOG_INODE_EXISTS, + inode, log_mode == LOG_INODE_EXISTS, logged_isize); } else { copy_extent_buffer(dst_path->nodes[0], src, dst_offset, @@ -5913,9 +5917,13 @@ static int log_new_dir_dentries(struct btrfs_trans_handle *trans, struct btrfs_inode *curr_inode = start_inode; int ret = 0; + trace_btrfs_log_new_dir_dentries_enter(trans, start_inode); + path = btrfs_alloc_path(); - if (!path) - return -ENOMEM; + if (!path) { + ret = -ENOMEM; + goto out; + } /* Pairs with btrfs_add_delayed_iput below. */ ihold(&curr_inode->vfs_inode); @@ -6034,6 +6042,8 @@ out: kfree(dir_elem); } + trace_btrfs_log_new_dir_dentries_exit(trans, start_inode, ret); + return ret; } @@ -6126,6 +6136,9 @@ static int add_conflicting_inode(struct btrfs_trans_handle *trans, { struct btrfs_ino_list *ino_elem; struct btrfs_inode *inode; + int ret = 0; + + trace_btrfs_add_conflicting_inode_enter(trans, ctx, ino, parent); /* * It's rare to have a lot of conflicting inodes, in practice it is not @@ -6134,8 +6147,10 @@ static int add_conflicting_inode(struct btrfs_trans_handle *trans, * LOG_INODE_EXISTS mode) and slow down other fsyncs or transaction * commits. */ - if (ctx->num_conflict_inodes >= MAX_CONFLICT_INODES) - return BTRFS_LOG_FORCE_COMMIT; + if (ctx->num_conflict_inodes >= MAX_CONFLICT_INODES) { + ret = BTRFS_LOG_FORCE_COMMIT; + goto out; + } inode = btrfs_iget_logging(ino, root); /* @@ -6159,26 +6174,27 @@ static int add_conflicting_inode(struct btrfs_trans_handle *trans, * some inode from it to some other directory). */ if (IS_ERR(inode)) { - int ret = PTR_ERR(inode); - + ret = PTR_ERR(inode); if (ret != -ENOENT) - return ret; + goto out; ret = conflicting_inode_is_dir(root, ino, path); /* Not a directory or we got an error. */ if (ret <= 0) - return ret; + goto out; /* Conflicting inode is a directory, so we'll log its parent. */ ino_elem = kmalloc_obj(*ino_elem, GFP_NOFS); - if (!ino_elem) - return -ENOMEM; + if (!ino_elem) { + ret = -ENOMEM; + goto out; + } ino_elem->ino = ino; ino_elem->parent = parent; list_add_tail(&ino_elem->list, &ctx->conflict_inodes); ctx->num_conflict_inodes++; - - return 0; + ret = 0; + goto out; } /* @@ -6218,25 +6234,31 @@ static int add_conflicting_inode(struct btrfs_trans_handle *trans, */ if (!need_log_inode(trans, inode)) { btrfs_add_delayed_iput(inode); - return 0; + goto out; } if (!can_log_conflicting_inode(trans, inode)) { btrfs_add_delayed_iput(inode); - return BTRFS_LOG_FORCE_COMMIT; + ret = BTRFS_LOG_FORCE_COMMIT; + goto out; } btrfs_add_delayed_iput(inode); ino_elem = kmalloc_obj(*ino_elem, GFP_NOFS); - if (!ino_elem) - return -ENOMEM; + if (!ino_elem) { + ret = -ENOMEM; + goto out; + } ino_elem->ino = ino; ino_elem->parent = parent; list_add_tail(&ino_elem->list, &ctx->conflict_inodes); ctx->num_conflict_inodes++; - return 0; +out: + trace_btrfs_add_conflicting_inode_exit(trans, ctx, ino, parent, ret); + + return ret; } static int log_conflicting_inodes(struct btrfs_trans_handle *trans, @@ -6254,7 +6276,15 @@ static int log_conflicting_inodes(struct btrfs_trans_handle *trans, if (ctx->logging_conflict_inodes) return 0; + /* + * Avoid any work if no conflicting inodes and emitting the trace event + * which only adds noise and it's useless if there are no inodes. + */ + if (list_empty(&ctx->conflict_inodes)) + return 0; + ctx->logging_conflict_inodes = true; + trace_btrfs_log_conflicting_inodes_enter(trans, ctx); /* * New conflicting inodes may be found and added to the list while we @@ -6348,6 +6378,7 @@ static int log_conflicting_inodes(struct btrfs_trans_handle *trans, ctx->logging_conflict_inodes = false; if (ret) free_conflicting_inodes(ctx); + trace_btrfs_log_conflicting_inodes_exit(trans, ctx, ret); return ret; } @@ -6359,7 +6390,7 @@ static int copy_inode_items_to_log(struct btrfs_trans_handle *trans, struct btrfs_path *path, struct btrfs_path *dst_path, const u64 logged_isize, - const int inode_only, + const enum btrfs_log_mode log_mode, struct btrfs_log_ctx *ctx, bool *need_log_inode_item) { @@ -6415,7 +6446,7 @@ again: } ret = copy_items(trans, inode, dst_path, path, ins_start_slot, ins_nr, - inode_only, logged_isize, ctx); + log_mode, logged_isize, ctx); if (ret < 0) return ret; ins_nr = 0; @@ -6434,7 +6465,7 @@ again: goto next_slot; ret = copy_items(trans, inode, dst_path, path, ins_start_slot, - ins_nr, inode_only, logged_isize, ctx); + ins_nr, log_mode, logged_isize, ctx); if (ret < 0) return ret; ins_nr = 0; @@ -6451,7 +6482,7 @@ again: } ret = copy_items(trans, inode, dst_path, path, ins_start_slot, - ins_nr, inode_only, logged_isize, ctx); + ins_nr, log_mode, logged_isize, ctx); if (ret < 0) return ret; ins_nr = 1; @@ -6465,7 +6496,7 @@ next_slot: } if (ins_nr) { ret = copy_items(trans, inode, dst_path, path, - ins_start_slot, ins_nr, inode_only, + ins_start_slot, ins_nr, log_mode, logged_isize, ctx); if (ret < 0) return ret; @@ -6491,12 +6522,12 @@ next_key: } if (ins_nr) { ret = copy_items(trans, inode, dst_path, path, ins_start_slot, - ins_nr, inode_only, logged_isize, ctx); + ins_nr, log_mode, logged_isize, ctx); if (ret) return ret; } - if (inode_only == LOG_INODE_ALL && S_ISREG(inode->vfs_inode.i_mode)) { + if (log_mode == LOG_INODE_ALL && S_ISREG(inode->vfs_inode.i_mode)) { /* * Release the path because otherwise we might attempt to double * lock the same leaf with btrfs_log_prealloc_extents() below. @@ -6827,8 +6858,16 @@ static int log_new_delayed_dentries(struct btrfs_trans_handle *trans, */ lockdep_assert_not_held(&inode->log_mutex); - ASSERT(!ctx->logging_new_delayed_dentries, - "ctx->logging_new_delayed_dentries=%d", ctx->logging_new_delayed_dentries); + ASSERT(!ctx->logging_new_delayed_dentries); + + /* + * Return early if empty list, avoid emitting redundant trace events + * that generate noise only. + */ + if (list_empty(delayed_ins_list)) + return 0; + + trace_btrfs_log_new_delayed_dentries_enter(trans, inode); ctx->logging_new_delayed_dentries = true; list_for_each_entry(item, delayed_ins_list, log_list) { @@ -6871,6 +6910,7 @@ static int log_new_delayed_dentries(struct btrfs_trans_handle *trans, ctx->log_new_dentries = orig_log_new_dentries; ctx->logging_new_delayed_dentries = false; + trace_btrfs_log_new_delayed_dentries_exit(trans, inode, ret); return ret; } @@ -6891,11 +6931,11 @@ static int log_new_delayed_dentries(struct btrfs_trans_handle *trans, */ static int btrfs_log_inode(struct btrfs_trans_handle *trans, struct btrfs_inode *inode, - int inode_only, + enum btrfs_log_mode log_mode, struct btrfs_log_ctx *ctx) { struct btrfs_path *path; - struct btrfs_path *dst_path; + struct btrfs_path *dst_path = NULL; struct btrfs_key min_key; struct btrfs_key max_key; struct btrfs_root *log = inode->root->log_root; @@ -6911,13 +6951,17 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, LIST_HEAD(delayed_ins_list); LIST_HEAD(delayed_del_list); + trace_btrfs_log_inode_enter(trans, inode, ctx, log_mode); + path = btrfs_alloc_path(); - if (!path) - return -ENOMEM; + if (!path) { + ret = -ENOMEM; + goto out; + } dst_path = btrfs_alloc_path(); if (!dst_path) { - btrfs_free_path(path); - return -ENOMEM; + ret = -ENOMEM; + goto out; } min_key.objectid = ino; @@ -6931,13 +6975,13 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, if (S_ISDIR(inode->vfs_inode.i_mode) || (!test_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags) && - inode_only >= LOG_INODE_EXISTS)) + log_mode >= LOG_INODE_EXISTS)) max_key.type = BTRFS_XATTR_ITEM_KEY; else max_key.type = (u8)-1; max_key.offset = (u64)-1; - if (S_ISDIR(inode->vfs_inode.i_mode) && inode_only == LOG_INODE_ALL) + if (S_ISDIR(inode->vfs_inode.i_mode) && log_mode == LOG_INODE_ALL) full_dir_logging = true; /* @@ -6988,7 +7032,7 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, * for symlinks). */ if (S_ISLNK(inode->vfs_inode.i_mode)) - inode_only = LOG_INODE_ALL; + log_mode = LOG_INODE_ALL; /* * Before logging the inode item, cache the value returned by @@ -7023,7 +7067,7 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, ret = drop_inode_items(trans, log, path, inode, BTRFS_XATTR_ITEM_KEY); } else { - if (inode_only == LOG_INODE_EXISTS) { + if (log_mode == LOG_INODE_EXISTS) { /* * Make sure the new inode item we write to the log has * the same isize as the current one (if it exists). @@ -7043,7 +7087,7 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, } if (test_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags)) { - if (inode_only == LOG_INODE_EXISTS) { + if (log_mode == LOG_INODE_EXISTS) { max_key.type = BTRFS_XATTR_ITEM_KEY; if (ctx->logged_before) ret = drop_inode_items(trans, log, path, @@ -7059,15 +7103,15 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, } } else if (test_and_clear_bit(BTRFS_INODE_COPY_EVERYTHING, &inode->runtime_flags) || - inode_only == LOG_INODE_EXISTS) { - if (inode_only == LOG_INODE_ALL) + log_mode == LOG_INODE_EXISTS) { + if (log_mode == LOG_INODE_ALL) fast_search = true; max_key.type = BTRFS_XATTR_ITEM_KEY; if (ctx->logged_before) ret = drop_inode_items(trans, log, path, inode, max_key.type); } else { - if (inode_only == LOG_INODE_ALL) + if (log_mode == LOG_INODE_ALL) fast_search = true; inode_item_dropped = false; goto log_extents; @@ -7102,8 +7146,7 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, ret = copy_inode_items_to_log(trans, inode, &min_key, &max_key, path, dst_path, logged_isize, - inode_only, ctx, - &need_log_inode_item); + log_mode, ctx, &need_log_inode_item); if (ret) goto out_unlock; @@ -7146,7 +7189,7 @@ log_extents: ret = btrfs_log_changed_extents(trans, inode, dst_path, ctx); if (ret) goto out_unlock; - } else if (inode_only == LOG_INODE_ALL) { + } else if (log_mode == LOG_INODE_ALL) { struct extent_map *em, *n; write_lock(&em_tree->lock); @@ -7202,7 +7245,7 @@ log_extents: * a power failure unless the log was synced as part of an fsync * against any other unrelated inode. */ - if (!ctx->logging_new_name && inode_only != LOG_INODE_EXISTS) + if (!ctx->logging_new_name && log_mode != LOG_INODE_EXISTS) inode->last_log_commit = inode->last_sub_trans; spin_unlock(&inode->lock); @@ -7210,7 +7253,7 @@ log_extents: * Reset the last_reflink_trans so that the next fsync does not need to * go through the slower path when logging extents and their checksums. */ - if (inode_only == LOG_INODE_ALL) + if (log_mode == LOG_INODE_ALL) inode->last_reflink_trans = 0; out_unlock: @@ -7233,6 +7276,8 @@ out: &delayed_del_list); } + trace_btrfs_log_inode_exit(trans, inode, ret); + return ret; } @@ -7246,9 +7291,13 @@ static int btrfs_log_all_parents(struct btrfs_trans_handle *trans, struct btrfs_root *root = inode->root; const u64 ino = btrfs_ino(inode); + trace_btrfs_log_all_parents_enter(trans, inode); + path = btrfs_alloc_path(); - if (!path) - return -ENOMEM; + if (!path) { + ret = -ENOMEM; + goto out; + } path->skip_locking = true; path->search_commit_root = true; @@ -7257,7 +7306,7 @@ static int btrfs_log_all_parents(struct btrfs_trans_handle *trans, key.offset = 0; ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); if (ret < 0) - return ret; + goto out; while (true) { struct extent_buffer *leaf = path->nodes[0]; @@ -7269,9 +7318,11 @@ static int btrfs_log_all_parents(struct btrfs_trans_handle *trans, if (slot >= btrfs_header_nritems(leaf)) { ret = btrfs_next_leaf(root, path); if (ret < 0) - return ret; - if (ret > 0) + goto out; + if (ret > 0) { + ret = 0; break; + } continue; } @@ -7324,8 +7375,10 @@ static int btrfs_log_all_parents(struct btrfs_trans_handle *trans, * at both parents and the old parent B would still * exist. */ - if (IS_ERR(dir_inode)) - return PTR_ERR(dir_inode); + if (IS_ERR(dir_inode)) { + ret = PTR_ERR(dir_inode); + goto out; + } if (!need_log_inode(trans, dir_inode)) { btrfs_add_delayed_iput(dir_inode); @@ -7338,11 +7391,14 @@ static int btrfs_log_all_parents(struct btrfs_trans_handle *trans, ret = log_new_dir_dentries(trans, dir_inode, ctx); btrfs_add_delayed_iput(dir_inode); if (ret) - return ret; + goto out; } path->slots[0]++; } - return 0; +out: + trace_btrfs_log_all_parents_exit(trans, inode, ret); + + return ret; } static int log_new_ancestors(struct btrfs_trans_handle *trans, @@ -7457,16 +7513,22 @@ static int log_all_new_ancestors(struct btrfs_trans_handle *trans, struct btrfs_key search_key; int ret; + trace_btrfs_log_all_new_ancestors_enter(trans, inode); + /* * For a single hard link case, go through a fast path that does not * need to iterate the fs/subvolume tree. */ - if (inode->vfs_inode.i_nlink < 2) - return log_new_ancestors_fast(trans, inode, parent, ctx); + if (inode->vfs_inode.i_nlink < 2) { + ret = log_new_ancestors_fast(trans, inode, parent, ctx); + goto out; + } path = btrfs_alloc_path(); - if (!path) - return -ENOMEM; + if (!path) { + ret = -ENOMEM; + goto out; + } search_key.objectid = ino; search_key.type = BTRFS_INODE_REF_KEY; @@ -7474,7 +7536,7 @@ static int log_all_new_ancestors(struct btrfs_trans_handle *trans, again: ret = btrfs_search_slot(NULL, root, &search_key, path, 0, 0); if (ret < 0) - return ret; + goto out; if (ret == 0) path->slots[0]++; @@ -7486,9 +7548,11 @@ again: if (slot >= btrfs_header_nritems(leaf)) { ret = btrfs_next_leaf(root, path); if (ret < 0) - return ret; - if (ret > 0) + goto out; + if (ret > 0) { + ret = 0; break; + } continue; } @@ -7504,8 +7568,10 @@ again: * this loop, etc). So just return some error to fallback to * a transaction commit. */ - if (found_key.type == BTRFS_INODE_EXTREF_KEY) - return -EMLINK; + if (found_key.type == BTRFS_INODE_EXTREF_KEY) { + ret = -EMLINK; + goto out; + } /* * Logging ancestors needs to do more searches on the fs/subvol @@ -7517,11 +7583,13 @@ again: ret = log_new_ancestors(trans, root, path, ctx); if (ret) - return ret; + goto out; btrfs_release_path(path); goto again; } - return 0; +out: + trace_btrfs_log_all_new_ancestors_exit(trans, inode, ret); + return ret; } /* @@ -7533,7 +7601,7 @@ again: static int btrfs_log_inode_parent(struct btrfs_trans_handle *trans, struct btrfs_inode *inode, struct dentry *parent, - int inode_only, + enum btrfs_log_mode log_mode, struct btrfs_log_ctx *ctx) { struct btrfs_root *root = inode->root; @@ -7541,29 +7609,39 @@ static int btrfs_log_inode_parent(struct btrfs_trans_handle *trans, int ret = 0; bool log_dentries; - if (btrfs_test_opt(fs_info, NOTREELOG)) - return BTRFS_LOG_FORCE_COMMIT; + trace_btrfs_log_inode_parent_enter(trans, inode); - if (btrfs_root_refs(&root->root_item) == 0) - return BTRFS_LOG_FORCE_COMMIT; + if (btrfs_test_opt(fs_info, NOTREELOG)) { + ret = BTRFS_LOG_FORCE_COMMIT; + goto out; + } + + if (btrfs_root_refs(&root->root_item) == 0) { + ret = BTRFS_LOG_FORCE_COMMIT; + goto out; + } /* * If we're logging an inode from a subvolume created in the current * transaction we must force a commit since the root is not persisted. */ - if (btrfs_root_generation(&root->root_item) == trans->transid) - return BTRFS_LOG_FORCE_COMMIT; + if (btrfs_root_generation(&root->root_item) == trans->transid) { + ret = BTRFS_LOG_FORCE_COMMIT; + goto out; + } /* Skip already logged inodes and without new extents. */ if (btrfs_inode_in_log(inode, trans->transid) && - list_empty(&ctx->ordered_extents)) - return BTRFS_NO_LOG_SYNC; + list_empty(&ctx->ordered_extents)) { + ret = BTRFS_NO_LOG_SYNC; + goto out; + } ret = start_log_trans(trans, root, ctx); if (ret) - return ret; + goto out; - ret = btrfs_log_inode(trans, inode, inode_only, ctx); + ret = btrfs_log_inode(trans, inode, log_mode, ctx); if (ret) goto end_trans; @@ -7649,6 +7727,9 @@ end_trans: btrfs_remove_log_ctx(root, ctx); btrfs_end_log_trans(root); +out: + trace_btrfs_log_inode_parent_exit(trans, inode, ret); + return ret; } @@ -7872,6 +7953,8 @@ void btrfs_record_unlink_dir(struct btrfs_trans_handle *trans, struct btrfs_inode *dir, struct btrfs_inode *inode, bool for_rename) { + trace_btrfs_record_unlink_dir(trans, dir, inode, for_rename); + /* * when we're logging a file, if it hasn't been renamed * or unlinked, and its inode is fully committed on disk, @@ -7934,6 +8017,8 @@ void btrfs_record_unlink_dir(struct btrfs_trans_handle *trans, void btrfs_record_snapshot_destroy(struct btrfs_trans_handle *trans, struct btrfs_inode *dir) { + trace_btrfs_record_snapshot_destroy(trans, dir); + mutex_lock(&dir->log_mutex); dir->last_unlink_trans = trans->transid; mutex_unlock(&dir->log_mutex); @@ -7954,6 +8039,8 @@ void btrfs_record_snapshot_destroy(struct btrfs_trans_handle *trans, void btrfs_record_new_subvolume(const struct btrfs_trans_handle *trans, struct btrfs_inode *dir) { + trace_btrfs_record_new_subvolume(trans, dir); + mutex_lock(&dir->log_mutex); dir->last_unlink_trans = trans->transid; mutex_unlock(&dir->log_mutex); @@ -7986,6 +8073,8 @@ void btrfs_log_new_name(struct btrfs_trans_handle *trans, bool log_pinned = false; int ret; + trace_btrfs_log_new_name_enter(trans, inode, old_dir, old_dir_index); + /* The inode has a new name (ref/extref), so make sure we log it. */ set_bit(BTRFS_INODE_COPY_EVERYTHING, &inode->runtime_flags); @@ -8008,7 +8097,7 @@ void btrfs_log_new_name(struct btrfs_trans_handle *trans, goto out; } else if (ret == 0) { if (!old_dir) - return; + goto out; /* * If the inode was not logged and we are doing a rename (old_dir is not * NULL), check if old_dir was logged - if it was not we can return and @@ -8018,7 +8107,7 @@ void btrfs_log_new_name(struct btrfs_trans_handle *trans, if (ret < 0) goto out; else if (ret == 0) - return; + goto out; } ret = 0; @@ -8117,6 +8206,7 @@ void btrfs_log_new_name(struct btrfs_trans_handle *trans, btrfs_log_inode_parent(trans, inode, parent, LOG_INODE_EXISTS, &ctx); ASSERT(list_empty(&ctx.conflict_inodes)); out: + trace_btrfs_log_new_name_exit(trans, inode, old_dir, ret); /* * If an error happened mark the log for a full commit because it's not * consistent and up to date or we couldn't find out if one of the diff --git a/fs/btrfs/tree-log.h b/fs/btrfs/tree-log.h index 4a626dc6a58b..81ab5eeeb974 100644 --- a/fs/btrfs/tree-log.h +++ b/fs/btrfs/tree-log.h @@ -11,6 +11,13 @@ #include #include "transaction.h" +enum btrfs_log_mode { + /* Log everything about an inode. */ + LOG_INODE_ALL, + /* Log just enough to recreate the inode during log replay. */ + LOG_INODE_EXISTS, +}; + struct inode; struct dentry; struct btrfs_ordered_extent; diff --git a/fs/btrfs/verity.c b/fs/btrfs/verity.c index 0062b3a55781..983365a73541 100644 --- a/fs/btrfs/verity.c +++ b/fs/btrfs/verity.c @@ -458,7 +458,7 @@ static int rollback_verity(struct btrfs_inode *inode) if (ret) { btrfs_handle_fs_error(root->fs_info, ret, "failed to drop verity items in rollback %llu", - (u64)inode->vfs_inode.i_ino); + inode->vfs_inode.i_ino); goto out; } @@ -472,7 +472,7 @@ static int rollback_verity(struct btrfs_inode *inode) trans = NULL; btrfs_handle_fs_error(root->fs_info, ret, "failed to start transaction in verity rollback %llu", - (u64)inode->vfs_inode.i_ino); + inode->vfs_inode.i_ino); goto out; } inode->ro_flags &= ~BTRFS_INODE_RO_VERITY; diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index a88e68f90564..6eab4cc73ce4 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -963,6 +963,11 @@ static noinline struct btrfs_device *device_list_add(const char *path, devid, btrfs_dev_name(device), path, current->comm, task_pid_nr(current)); + } else { + btrfs_info(NULL, + "missing devid %llu re-appeared at %s scanned by %s (%d)", + devid, path, current->comm, + task_pid_nr(current)); } name = kstrdup(path, GFP_NOFS); @@ -1369,9 +1374,9 @@ struct btrfs_super_block *btrfs_read_disk_super(struct block_device *bdev, (bytenr + BTRFS_SUPER_INFO_SIZE) >> PAGE_SHIFT); } - filemap_invalidate_lock(mapping); + filemap_invalidate_lock_shared(mapping); page = read_cache_page_gfp(mapping, bytenr >> PAGE_SHIFT, GFP_NOFS); - filemap_invalidate_unlock(mapping); + filemap_invalidate_unlock_shared(mapping); if (IS_ERR(page)) return ERR_CAST(page); @@ -2286,6 +2291,38 @@ void btrfs_scratch_superblocks(struct btrfs_fs_info *fs_info, struct btrfs_devic update_dev_time(rcu_dereference_raw(device->name)); } +int btrfs_remove_dev_stat_item(struct btrfs_trans_handle *trans, u64 devid) +{ + BTRFS_PATH_AUTO_RELEASE(path); + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_root *dev_root = fs_info->dev_root; + struct btrfs_key key; + int ret; + + key.objectid = BTRFS_DEV_STATS_OBJECTID; + key.type = BTRFS_PERSISTENT_ITEM_KEY; + key.offset = devid; + + ret = btrfs_search_slot(trans, dev_root, &key, &path, -1, 1); + if (ret < 0) { + btrfs_warn(fs_info, + "error %d while searching for dev_stats item for devid %llu", + ret, devid); + return ret; + } + /* The dev stats item does not exist, nothing to bother. */ + if (ret > 0) + return 0; + ret = btrfs_del_item(trans, dev_root, &path); + if (ret < 0) { + btrfs_warn(fs_info, + "error %d while deleting dev_stats item for devid %llu", + ret, devid); + return ret; + } + return 0; +} + int btrfs_rm_device(struct btrfs_fs_info *fs_info, struct btrfs_dev_lookup_args *args, struct file **bdev_file) @@ -2365,6 +2402,12 @@ int btrfs_rm_device(struct btrfs_fs_info *fs_info, return ret; } + ret = btrfs_remove_dev_stat_item(trans, device->devid); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + btrfs_end_transaction(trans); + return ret; + } clear_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state); btrfs_scrub_cancel_dev(device); @@ -2889,6 +2932,12 @@ int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path device->commit_total_bytes = device->total_bytes; set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state); clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state); + + /* + * Increase dev_stats_ccnt so that corresponding DEV_STATS item can be + * created at the next transaction commit. + */ + atomic_inc(&device->dev_stats_ccnt); device->dev_stats_valid = 1; set_blocksize(device->bdev_file, BTRFS_BDEV_BLOCKSIZE); @@ -3718,7 +3767,11 @@ static int btrfs_may_alloc_data_chunk(struct btrfs_fs_info *fs_info, u64 chunk_type; cache = btrfs_lookup_block_group(fs_info, chunk_offset); - ASSERT(cache); + if (unlikely(!cache)) { + btrfs_err(fs_info, "balance: chunk at bytenr %llu has no corresponding block group", + chunk_offset); + return -EUCLEAN; + } chunk_type = cache->flags; btrfs_put_block_group(cache); @@ -3957,16 +4010,21 @@ static bool chunk_profiles_filter(u64 chunk_type, struct btrfs_balance_args *bar return true; } -static bool chunk_usage_range_filter(struct btrfs_fs_info *fs_info, u64 chunk_offset, - struct btrfs_balance_args *bargs) +static int chunk_usage_range_filter(struct btrfs_fs_info *fs_info, u64 chunk_offset, + struct btrfs_balance_args *bargs) { struct btrfs_block_group *cache; u64 chunk_used; u64 user_thresh_min; u64 user_thresh_max; - bool ret = true; + int ret = 1; cache = btrfs_lookup_block_group(fs_info, chunk_offset); + if (unlikely(!cache)) { + btrfs_err(fs_info, "balance: chunk at bytenr %llu has no corresponding block group", + chunk_offset); + return -EUCLEAN; + } chunk_used = cache->used; if (bargs->usage_min == 0) @@ -3982,20 +4040,25 @@ static bool chunk_usage_range_filter(struct btrfs_fs_info *fs_info, u64 chunk_of user_thresh_max = mult_perc(cache->length, bargs->usage_max); if (user_thresh_min <= chunk_used && chunk_used < user_thresh_max) - ret = false; + ret = 0; btrfs_put_block_group(cache); return ret; } -static bool chunk_usage_filter(struct btrfs_fs_info *fs_info, u64 chunk_offset, - struct btrfs_balance_args *bargs) +static int chunk_usage_filter(struct btrfs_fs_info *fs_info, u64 chunk_offset, + struct btrfs_balance_args *bargs) { struct btrfs_block_group *cache; u64 chunk_used, user_thresh; - bool ret = true; + int ret = 1; cache = btrfs_lookup_block_group(fs_info, chunk_offset); + if (unlikely(!cache)) { + btrfs_err(fs_info, "balance: chunk at bytenr %llu has no corresponding block group", + chunk_offset); + return -EUCLEAN; + } chunk_used = cache->used; if (bargs->usage_min == 0) @@ -4006,7 +4069,7 @@ static bool chunk_usage_filter(struct btrfs_fs_info *fs_info, u64 chunk_offset, user_thresh = mult_perc(cache->length, bargs->usage); if (chunk_used < user_thresh) - ret = false; + ret = 0; btrfs_put_block_group(cache); return ret; @@ -4111,8 +4174,8 @@ static bool chunk_soft_convert_filter(u64 chunk_type, struct btrfs_balance_args return false; } -static bool should_balance_chunk(struct extent_buffer *leaf, struct btrfs_chunk *chunk, - u64 chunk_offset) +static int should_balance_chunk(struct extent_buffer *leaf, struct btrfs_chunk *chunk, + u64 chunk_offset) { struct btrfs_fs_info *fs_info = leaf->fs_info; struct btrfs_balance_control *bctl = fs_info->balance_ctl; @@ -4145,12 +4208,22 @@ static bool should_balance_chunk(struct extent_buffer *leaf, struct btrfs_chunk } /* usage filter */ - if ((bargs->flags & BTRFS_BALANCE_ARGS_USAGE) && - chunk_usage_filter(fs_info, chunk_offset, bargs)) { - return false; - } else if ((bargs->flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) && - chunk_usage_range_filter(fs_info, chunk_offset, bargs)) { - return false; + if (bargs->flags & BTRFS_BALANCE_ARGS_USAGE) { + int ret2; + + ret2 = chunk_usage_filter(fs_info, chunk_offset, bargs); + if (ret2 < 0) + return ret2; + if (ret2) + return false; + } else if (bargs->flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) { + int ret2; + + ret2 = chunk_usage_range_filter(fs_info, chunk_offset, bargs); + if (ret2 < 0) + return ret2; + if (ret2) + return false; } /* devid filter */ @@ -4337,7 +4410,7 @@ static int __btrfs_balance(struct btrfs_fs_info *fs_info) u32 count_data = 0; u32 count_meta = 0; u32 count_sys = 0; - int chunk_reserved = 0; + bool chunk_reserved = false; struct remap_chunk_info *rci; unsigned int num_remap_chunks = 0; LIST_HEAD(remap_chunks); @@ -4430,6 +4503,10 @@ again: ret = should_balance_chunk(leaf, chunk, found_key.offset); btrfs_release_path(path); + if (ret < 0) { + mutex_unlock(&fs_info->reclaim_bgs_lock); + goto error; + } if (!ret) { mutex_unlock(&fs_info->reclaim_bgs_lock); goto loop; @@ -4502,7 +4579,7 @@ again: mutex_unlock(&fs_info->reclaim_bgs_lock); goto error; } else if (ret == 1) { - chunk_reserved = 1; + chunk_reserved = true; } } @@ -4763,7 +4840,7 @@ int btrfs_balance(struct btrfs_fs_info *fs_info, { u64 meta_target, data_target; u64 allowed; - int mixed = 0; + bool mixed = false; int ret; u64 num_devices; unsigned seq; @@ -4780,7 +4857,7 @@ int btrfs_balance(struct btrfs_fs_info *fs_info, allowed = btrfs_super_incompat_flags(fs_info->super_copy); if (allowed & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS) - mixed = 1; + mixed = true; /* * In case of mixed groups both data and meta should be picked, @@ -6053,7 +6130,7 @@ struct btrfs_block_group *btrfs_create_chunk(struct btrfs_trans_handle *trans, lockdep_assert_held(&info->chunk_mutex); - if (!alloc_profile_is_valid(type, 0)) { + if (unlikely(!alloc_profile_is_valid(type, 0))) { DEBUG_WARN("invalid alloc profile for type %llu", type); return ERR_PTR(-EINVAL); } @@ -6064,7 +6141,7 @@ struct btrfs_block_group *btrfs_create_chunk(struct btrfs_trans_handle *trans, return ERR_PTR(-ENOSPC); } - if (!(type & BTRFS_BLOCK_GROUP_TYPE_MASK)) { + if (unlikely(!(type & BTRFS_BLOCK_GROUP_TYPE_MASK))) { btrfs_err(info, "invalid chunk type 0x%llx requested", type); DEBUG_WARN(); return ERR_PTR(-EINVAL); @@ -6234,7 +6311,7 @@ static noinline int init_first_rw_device(struct btrfs_trans_handle *trans) alloc_profile = btrfs_metadata_alloc_profile(fs_info); meta_space_info = btrfs_find_space_info(fs_info, alloc_profile); - if (!meta_space_info) { + if (unlikely(!meta_space_info)) { DEBUG_WARN(); return -EINVAL; } @@ -6244,7 +6321,7 @@ static noinline int init_first_rw_device(struct btrfs_trans_handle *trans) alloc_profile = btrfs_system_alloc_profile(fs_info); sys_space_info = btrfs_find_space_info(fs_info, alloc_profile); - if (!sys_space_info) { + if (unlikely(!sys_space_info)) { DEBUG_WARN(); return -EINVAL; } @@ -8137,8 +8214,8 @@ static int btrfs_device_init_dev_stats(struct btrfs_device *device, for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++) { if (item_size >= (1 + i) * sizeof(__le64)) - btrfs_dev_stat_set(device, i, - btrfs_dev_stats_value(eb, ptr, i)); + atomic_set(device->dev_stat_values + i, + btrfs_dev_stats_value(eb, ptr, i)); else btrfs_dev_stat_set(device, i, 0); } @@ -8179,6 +8256,37 @@ out: return ret; } +int btrfs_init_writeback_bio_size(struct btrfs_fs_info *fs_info) +{ + struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; + struct btrfs_device *device; + u32 writeback_bio_size = fs_info->sectorsize; + + mutex_lock(&fs_devices->device_list_mutex); + /* + * Let's take maximum over optimal request sizes for all devices. For + * RAID profiles writeback will submit stripe (64k) sized bios anyway + * so our value doesn't matter and for simple profiles this is a good + * approximation of sensible IO chunking. + */ + list_for_each_entry(device, &fs_devices->devices, dev_list) { + struct request_queue *queue; + unsigned int io_opt; + + if (!device->bdev || test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) + continue; + queue = bdev_get_queue(device->bdev); + io_opt = queue_io_opt(queue) ? : + queue_max_sectors(queue) << SECTOR_SHIFT; + writeback_bio_size = max(writeback_bio_size, io_opt); + } + mutex_unlock(&fs_devices->device_list_mutex); + + fs_info->writeback_bio_size = writeback_bio_size; + + return 0; +} + static int update_dev_stat_item(struct btrfs_trans_handle *trans, struct btrfs_device *device) { diff --git a/fs/btrfs/volumes.h b/fs/btrfs/volumes.h index 0082c166af91..63be45c3298c 100644 --- a/fs/btrfs/volumes.h +++ b/fs/btrfs/volumes.h @@ -784,6 +784,7 @@ int btrfs_get_dev_stats(struct btrfs_fs_info *fs_info, struct btrfs_ioctl_get_dev_stats *stats); int btrfs_init_devices_late(struct btrfs_fs_info *fs_info); int btrfs_init_dev_stats(struct btrfs_fs_info *fs_info); +int btrfs_init_writeback_bio_size(struct btrfs_fs_info *fs_info); int btrfs_run_dev_stats(struct btrfs_trans_handle *trans); void btrfs_rm_dev_replace_remove_srcdev(struct btrfs_device *srcdev); void btrfs_rm_dev_replace_free_srcdev(struct btrfs_device *srcdev); @@ -932,6 +933,7 @@ bool btrfs_first_pending_extent(struct btrfs_device *device, u64 start, u64 len, u64 *pending_start, u64 *pending_end); bool btrfs_find_hole_in_pending_extents(struct btrfs_device *device, u64 *start, u64 *len, u64 min_hole_size); +int btrfs_remove_dev_stat_item(struct btrfs_trans_handle *trans, u64 devid); #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS struct btrfs_io_context *alloc_btrfs_io_context(struct btrfs_fs_info *fs_info, diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index 16dd87aa06f2..97f06dd01693 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -131,8 +131,10 @@ static int sb_write_pointer(struct block_device *bdev, struct blk_zone *zones, u64 bytenr = ALIGN_DOWN(zone_end, BTRFS_SUPER_INFO_SIZE) - BTRFS_SUPER_INFO_SIZE; + filemap_invalidate_lock_shared(mapping); page[i] = read_cache_page_gfp(mapping, bytenr >> PAGE_SHIFT, GFP_NOFS); + filemap_invalidate_unlock_shared(mapping); if (IS_ERR(page[i])) { if (i == 1) btrfs_release_disk_super(super[0]); @@ -354,12 +356,33 @@ int btrfs_get_dev_zone_info_all_devices(struct btrfs_fs_info *fs_info) return ret; } +static int btrfs_get_max_active_zones(struct btrfs_device *device, + struct btrfs_zoned_device_info *zone_info) +{ + struct block_device *bdev = device->bdev; + int max_active_zones; + + if (unlikely(zone_info->nr_zones < BTRFS_MIN_ACTIVE_ZONES)) { + btrfs_err(device->fs_info, "zoned: not enough zones to mount filesystem: %u < %d", + zone_info->nr_zones, BTRFS_MIN_ACTIVE_ZONES); + return -EINVAL; + } + + max_active_zones = min_not_zero(bdev_max_active_zones(bdev), + bdev_max_open_zones(bdev)); + if (max_active_zones == 0) + max_active_zones = min(zone_info->nr_zones / 4, + BTRFS_DEFAULT_MAX_ACTIVE_ZONES); + + zone_info->max_active_zones = max(max_active_zones, BTRFS_MIN_ACTIVE_ZONES); + return 0; +} + int btrfs_get_dev_zone_info(struct btrfs_device *device, bool populate_cache) { struct btrfs_fs_info *fs_info = device->fs_info; struct btrfs_zoned_device_info *zone_info = NULL; struct block_device *bdev = device->bdev; - unsigned int max_active_zones; unsigned int nactive; sector_t nr_sectors; sector_t sector = 0; @@ -424,19 +447,9 @@ int btrfs_get_dev_zone_info(struct btrfs_device *device, bool populate_cache) if (!IS_ALIGNED(nr_sectors, zone_sectors)) zone_info->nr_zones++; - max_active_zones = min_not_zero(bdev_max_active_zones(bdev), - bdev_max_open_zones(bdev)); - if (!max_active_zones && zone_info->nr_zones > BTRFS_DEFAULT_MAX_ACTIVE_ZONES) - max_active_zones = BTRFS_DEFAULT_MAX_ACTIVE_ZONES; - if (max_active_zones && max_active_zones < BTRFS_MIN_ACTIVE_ZONES) { - btrfs_err(fs_info, -"zoned: %s: max active zones %u is too small, need at least %u active zones", - rcu_dereference(device->name), max_active_zones, - BTRFS_MIN_ACTIVE_ZONES); - ret = -EINVAL; + ret = btrfs_get_max_active_zones(device, zone_info); + if (ret) goto out; - } - zone_info->max_active_zones = max_active_zones; zone_info->seq_zones = bitmap_zalloc(zone_info->nr_zones, GFP_KERNEL); if (!zone_info->seq_zones) { @@ -517,26 +530,29 @@ int btrfs_get_dev_zone_info(struct btrfs_device *device, bool populate_cache) goto out; } - if (max_active_zones) { - if (unlikely(nactive > max_active_zones)) { - if (bdev_max_active_zones(bdev) == 0) { - max_active_zones = 0; - zone_info->max_active_zones = 0; - goto validate; - } + if (unlikely(nactive > zone_info->max_active_zones)) { + if (bdev_max_active_zones(bdev) > 0) { btrfs_err(device->fs_info, - "zoned: %u active zones on %s exceeds max_active_zones %u", - nactive, rcu_dereference(device->name), - max_active_zones); + "zoned: %u active zones on %s exceeds max_active_zones %u", + nactive, rcu_dereference(device->name), + zone_info->max_active_zones); ret = -EIO; goto out; } + + /* + * This is for backward compatibility with old filesystems that + * have a lot of active zones because the device doesn't report + * a maximum number of zones and we previously didn't care for + * the limit. + */ + zone_info->max_active_zones = 0; + } else { atomic_set(&zone_info->active_zones_left, - max_active_zones - nactive); + zone_info->max_active_zones - nactive); set_bit(BTRFS_FS_ACTIVE_ZONE_TRACKING, &fs_info->flags); } -validate: /* Validate superblock log */ nr_zones = BTRFS_NR_SB_LOG_ZONES; for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) { @@ -1311,7 +1327,7 @@ static int btrfs_load_zone_info(struct btrfs_fs_info *fs_info, int zone_idx, { struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; struct btrfs_device *device; - int dev_replace_is_ongoing = 0; + bool dev_replace_is_ongoing = false; unsigned int nofs_flag; struct blk_zone zone; int ret; @@ -2763,7 +2779,6 @@ void btrfs_zoned_reserve_data_reloc_bg(struct btrfs_fs_info *fs_info) struct btrfs_block_group *bg; struct list_head *bg_list; u64 alloc_flags; - bool first = true; bool did_chunk_alloc = false; int index; int ret; @@ -2780,18 +2795,13 @@ void btrfs_zoned_reserve_data_reloc_bg(struct btrfs_fs_info *fs_info) alloc_flags = btrfs_get_alloc_profile(fs_info, space_info->flags); index = btrfs_bg_flags_to_raid_index(alloc_flags); - /* Scan the data space_info to find empty block groups. Take the second one. */ again: bg_list = &space_info->block_groups[index]; list_for_each_entry(bg, bg_list, list) { + if (bg->alloc_offset != 0) continue; - if (first) { - first = false; - continue; - } - if (space_info == data_sinfo) { /* Migrate the block group to the data relocation space_info. */ struct btrfs_space_info *reloc_sinfo = data_sinfo->sub_group[0]; @@ -2803,8 +2813,6 @@ again: down_write(&space_info->groups_sem); list_del_init(&bg->list); - /* We can assume this as we choose the second empty one. */ - ASSERT(!list_empty(&space_info->block_groups[index])); up_write(&space_info->groups_sem); spin_lock(&space_info->lock); @@ -2849,7 +2857,6 @@ again: * We allocated a new block group in the data relocation space_info. We * can take that one. */ - first = false; did_chunk_alloc = true; goto again; } diff --git a/fs/dlm/lowcomms.c b/fs/dlm/lowcomms.c index 5b6142787919..2aff1c7c17de 100644 --- a/fs/dlm/lowcomms.c +++ b/fs/dlm/lowcomms.c @@ -271,7 +271,8 @@ static struct connection *__find_con(int nodeid, int r) { struct connection *con; - hlist_for_each_entry_rcu(con, &connection_hash[r], list) { + hlist_for_each_entry_srcu(con, &connection_hash[r], list, + srcu_read_lock_held(&connections_srcu)) { if (con->nodeid == nodeid) return con; } @@ -426,7 +427,8 @@ static int addr_to_nodeid(struct sockaddr_storage *addr, int *nodeid, idx = srcu_read_lock(&connections_srcu); for (i = 0; i < CONN_HASH_SIZE; i++) { - hlist_for_each_entry_rcu(con, &connection_hash[i], list) { + hlist_for_each_entry_srcu(con, &connection_hash[i], list, + srcu_read_lock_held(&connections_srcu)) { WARN_ON_ONCE(!con->addr_count); spin_lock(&con->addrs_lock); @@ -1729,7 +1731,8 @@ void dlm_lowcomms_shutdown(void) idx = srcu_read_lock(&connections_srcu); for (i = 0; i < CONN_HASH_SIZE; i++) { - hlist_for_each_entry_rcu(con, &connection_hash[i], list) { + hlist_for_each_entry_srcu(con, &connection_hash[i], list, + srcu_read_lock_held(&connections_srcu)) { shutdown_connection(con, true); stop_connection_io(con); flush_workqueue(process_workqueue); @@ -1968,7 +1971,8 @@ void dlm_lowcomms_exit(void) idx = srcu_read_lock(&connections_srcu); for (i = 0; i < CONN_HASH_SIZE; i++) { - hlist_for_each_entry_rcu(con, &connection_hash[i], list) { + hlist_for_each_entry_srcu(con, &connection_hash[i], list, + srcu_read_lock_held(&connections_srcu)) { spin_lock(&connections_lock); hlist_del_rcu(&con->list); spin_unlock(&connections_lock); diff --git a/fs/dlm/memory.c b/fs/dlm/memory.c index 5c35cc67aca4..ee55994ce90d 100644 --- a/fs/dlm/memory.c +++ b/fs/dlm/memory.c @@ -48,8 +48,10 @@ int __init dlm_memory_init(void) if (!rsb_cache) goto rsb; - cb_cache = kmem_cache_create("dlm_cb", sizeof(struct dlm_callback), + cb_cache = kmem_cache_create_usercopy("dlm_cb", sizeof(struct dlm_callback), __alignof__(struct dlm_callback), 0, + offsetof(struct dlm_callback, lvbptr), + sizeof_field(struct dlm_callback, lvbptr), NULL); if (!cb_cache) goto cb; diff --git a/fs/dlm/midcomms.c b/fs/dlm/midcomms.c index d54bdd8fc4f2..8964164600d2 100644 --- a/fs/dlm/midcomms.c +++ b/fs/dlm/midcomms.c @@ -275,7 +275,8 @@ static struct midcomms_node *__find_node(int nodeid, int r) { struct midcomms_node *node; - hlist_for_each_entry_rcu(node, &node_hash[r], hlist) { + hlist_for_each_entry_srcu(node, &node_hash[r], hlist, + srcu_read_lock_held(&nodes_srcu)) { if (node->nodeid == nodeid) return node; } @@ -355,6 +356,7 @@ int dlm_midcomms_addr(int nodeid, struct sockaddr_storage *addr) if (!node) return -ENOMEM; + node->debugfs = dlm_create_debug_comms_file(nodeid, node); node->nodeid = nodeid; spin_lock_init(&node->state_lock); spin_lock_init(&node->send_queue_lock); @@ -368,7 +370,6 @@ int dlm_midcomms_addr(int nodeid, struct sockaddr_storage *addr) hlist_add_head_rcu(&node->hlist, &node_hash[r]); spin_unlock_bh(&nodes_lock); - node->debugfs = dlm_create_debug_comms_file(nodeid, node); return 0; } @@ -968,10 +969,10 @@ static void midcomms_new_msg_cb(void *data) atomic_inc(&mh->node->send_queue_cnt); spin_lock_bh(&mh->node->send_queue_lock); + /* need to be locked with list_add_tail_rcu() because list is ordered */ + mh->seq = atomic_fetch_inc(&mh->node->seq_send); list_add_tail_rcu(&mh->list, &mh->node->send_queue); spin_unlock_bh(&mh->node->send_queue_lock); - - mh->seq = atomic_fetch_inc(&mh->node->seq_send); } static struct dlm_msg *dlm_midcomms_get_msg_3_2(struct dlm_mhandle *mh, int nodeid, @@ -1165,7 +1166,8 @@ void dlm_midcomms_exit(void) idx = srcu_read_lock(&nodes_srcu); for (i = 0; i < CONN_HASH_SIZE; i++) { - hlist_for_each_entry_rcu(node, &node_hash[i], hlist) { + hlist_for_each_entry_srcu(node, &node_hash[i], hlist, + srcu_read_lock_held(&nodes_srcu)) { dlm_delete_debug_comms_file(node->debugfs); spin_lock(&nodes_lock); @@ -1325,7 +1327,8 @@ void dlm_midcomms_version_wait(void) idx = srcu_read_lock(&nodes_srcu); for (i = 0; i < CONN_HASH_SIZE; i++) { - hlist_for_each_entry_rcu(node, &node_hash[i], hlist) { + hlist_for_each_entry_srcu(node, &node_hash[i], hlist, + srcu_read_lock_held(&nodes_srcu)) { ret = wait_event_timeout(node->shutdown_wait, node->version != DLM_VERSION_NOT_SET || node->state == DLM_CLOSED || @@ -1396,7 +1399,8 @@ void dlm_midcomms_shutdown(void) mutex_lock(&close_lock); idx = srcu_read_lock(&nodes_srcu); for (i = 0; i < CONN_HASH_SIZE; i++) { - hlist_for_each_entry_rcu(node, &node_hash[i], hlist) { + hlist_for_each_entry_srcu(node, &node_hash[i], hlist, + srcu_read_lock_held(&nodes_srcu)) { midcomms_shutdown(node); } } @@ -1404,7 +1408,8 @@ void dlm_midcomms_shutdown(void) dlm_lowcomms_shutdown(); for (i = 0; i < CONN_HASH_SIZE; i++) { - hlist_for_each_entry_rcu(node, &node_hash[i], hlist) { + hlist_for_each_entry_srcu(node, &node_hash[i], hlist, + srcu_read_lock_held(&nodes_srcu)) { midcomms_node_reset(node); } } diff --git a/fs/ext2/ext2.h b/fs/ext2/ext2.h index 3eb1f342645c..79f7b395258c 100644 --- a/fs/ext2/ext2.h +++ b/fs/ext2/ext2.h @@ -114,8 +114,6 @@ struct ext2_sb_info { */ spinlock_t s_lock; struct mb_cache *s_ea_block_cache; - struct dax_device *s_daxdev; - u64 s_dax_part_off; }; static inline spinlock_t * @@ -373,11 +371,9 @@ struct ext2_inode { #define EXT2_MOUNT_NO_UID32 0x000200 /* Disable 32-bit UIDs */ #define EXT2_MOUNT_XATTR_USER 0x004000 /* Extended user attributes */ #define EXT2_MOUNT_POSIX_ACL 0x008000 /* POSIX Access Control Lists */ -#define EXT2_MOUNT_XIP 0x010000 /* Obsolete, use DAX */ #define EXT2_MOUNT_USRQUOTA 0x020000 /* user quota */ #define EXT2_MOUNT_GRPQUOTA 0x040000 /* group quota */ #define EXT2_MOUNT_RESERVATION 0x080000 /* Preallocation */ -#define EXT2_MOUNT_DAX 0x100000 /* Direct Access */ #define clear_opt(o, opt) o &= ~EXT2_MOUNT_##opt diff --git a/fs/ext2/file.c b/fs/ext2/file.c index d9b1eb34694a..8dca9ec4cacd 100644 --- a/fs/ext2/file.c +++ b/fs/ext2/file.c @@ -21,7 +21,6 @@ #include #include -#include #include #include #include @@ -32,111 +31,6 @@ #include "acl.h" #include "trace.h" -#ifdef CONFIG_FS_DAX -static ssize_t ext2_dax_read_iter(struct kiocb *iocb, struct iov_iter *to) -{ - struct inode *inode = iocb->ki_filp->f_mapping->host; - ssize_t ret; - - if (!iov_iter_count(to)) - return 0; /* skip atime */ - - inode_lock_shared(inode); - ret = dax_iomap_rw(iocb, to, &ext2_iomap_ops); - inode_unlock_shared(inode); - - file_accessed(iocb->ki_filp); - return ret; -} - -static ssize_t ext2_dax_write_iter(struct kiocb *iocb, struct iov_iter *from) -{ - struct file *file = iocb->ki_filp; - struct inode *inode = file->f_mapping->host; - ssize_t ret; - - inode_lock(inode); - ret = generic_write_checks(iocb, from); - if (ret <= 0) - goto out_unlock; - ret = file_remove_privs(file); - if (ret) - goto out_unlock; - ret = file_update_time(file); - if (ret) - goto out_unlock; - - ret = dax_iomap_rw(iocb, from, &ext2_iomap_ops); - if (ret > 0 && iocb->ki_pos > i_size_read(inode)) { - i_size_write(inode, iocb->ki_pos); - mark_inode_dirty(inode); - } - -out_unlock: - inode_unlock(inode); - if (ret > 0) - ret = generic_write_sync(iocb, ret); - return ret; -} - -/* - * The lock ordering for ext2 DAX fault paths is: - * - * mmap_lock (MM) - * sb_start_pagefault (vfs, freeze) - * address_space->invalidate_lock - * address_space->i_mmap_rwsem or page_lock (mutually exclusive in DAX) - * ext2_inode_info->truncate_mutex - * - * The default page_lock and i_size verification done by non-DAX fault paths - * is sufficient because ext2 doesn't support hole punching. - */ -static vm_fault_t ext2_dax_fault(struct vm_fault *vmf) -{ - struct inode *inode = file_inode(vmf->vma->vm_file); - vm_fault_t ret; - bool write = (vmf->flags & FAULT_FLAG_WRITE) && - (vmf->vma->vm_flags & VM_SHARED); - - if (write) { - sb_start_pagefault(inode->i_sb); - file_update_time(vmf->vma->vm_file); - } - filemap_invalidate_lock_shared(inode->i_mapping); - - ret = dax_iomap_fault(vmf, 0, NULL, NULL, &ext2_iomap_ops); - - filemap_invalidate_unlock_shared(inode->i_mapping); - if (write) - sb_end_pagefault(inode->i_sb); - return ret; -} - -static const struct vm_operations_struct ext2_dax_vm_ops = { - .fault = ext2_dax_fault, - /* - * .huge_fault is not supported for DAX because allocation in ext2 - * cannot be reliably aligned to huge page sizes and so pmd faults - * will always fail and fail back to regular faults. - */ - .page_mkwrite = ext2_dax_fault, - .pfn_mkwrite = ext2_dax_fault, -}; - -static int ext2_file_mmap_prepare(struct vm_area_desc *desc) -{ - struct file *file = desc->file; - - if (!IS_DAX(file_inode(file))) - return generic_file_mmap_prepare(desc); - - file_accessed(file); - desc->vm_ops = &ext2_dax_vm_ops; - return 0; -} -#else -#define ext2_file_mmap_prepare generic_file_mmap_prepare -#endif /* * Called when filp is released. This happens when all file descriptors @@ -267,12 +161,15 @@ static ssize_t ext2_dio_write_iter(struct kiocb *iocb, struct iov_iter *from) endbyte = pos + status - 1; ret2 = filemap_write_and_wait_range(inode->i_mapping, pos, endbyte); - if (!ret2) + if (!ret2) { invalidate_mapping_pages(inode->i_mapping, pos >> PAGE_SHIFT, endbyte >> PAGE_SHIFT); - if (ret > 0) - generic_write_sync(iocb, ret); + if (ret > 0) + ret = generic_write_sync(iocb, ret); + } else { + ret = ret2; + } } out_unlock: @@ -285,10 +182,6 @@ out_unlock: static ssize_t ext2_file_read_iter(struct kiocb *iocb, struct iov_iter *to) { -#ifdef CONFIG_FS_DAX - if (IS_DAX(iocb->ki_filp->f_mapping->host)) - return ext2_dax_read_iter(iocb, to); -#endif if (iocb->ki_flags & IOCB_DIRECT) return ext2_dio_read_iter(iocb, to); @@ -297,10 +190,6 @@ static ssize_t ext2_file_read_iter(struct kiocb *iocb, struct iov_iter *to) static ssize_t ext2_file_write_iter(struct kiocb *iocb, struct iov_iter *from) { -#ifdef CONFIG_FS_DAX - if (IS_DAX(iocb->ki_filp->f_mapping->host)) - return ext2_dax_write_iter(iocb, from); -#endif if (iocb->ki_flags & IOCB_DIRECT) return ext2_dio_write_iter(iocb, from); @@ -321,7 +210,7 @@ const struct file_operations ext2_file_operations = { #ifdef CONFIG_COMPAT .compat_ioctl = ext2_compat_ioctl, #endif - .mmap_prepare = ext2_file_mmap_prepare, + .mmap_prepare = generic_file_mmap_prepare, .open = ext2_file_open, .release = ext2_release_file, .fsync = ext2_fsync, diff --git a/fs/ext2/inode.c b/fs/ext2/inode.c index 74aca5eb572d..29808629cce5 100644 --- a/fs/ext2/inode.c +++ b/fs/ext2/inode.c @@ -26,7 +26,6 @@ #include #include #include -#include #include #include #include @@ -741,27 +740,6 @@ static int ext2_get_blocks(struct inode *inode, goto cleanup; } - if (IS_DAX(inode)) { - /* - * We must unmap blocks before zeroing so that writeback cannot - * overwrite zeros with stale data from block device page cache. - */ - clean_bdev_aliases(inode->i_sb->s_bdev, - le32_to_cpu(chain[depth-1].key), - count); - /* - * block must be initialised before we put it in the tree - * so that it's not found by another thread before it's - * initialised - */ - err = sb_issue_zeroout(inode->i_sb, - le32_to_cpu(chain[depth-1].key), count, - GFP_KERNEL); - if (err) { - mutex_unlock(&ei->truncate_mutex); - goto cleanup; - } - } *new = true; ext2_splice_branch(inode, iblock, partial, indirect_blks, count); @@ -811,7 +789,6 @@ static int ext2_iomap_begin(struct inode *inode, loff_t offset, loff_t length, unsigned int blkbits = inode->i_blkbits; unsigned long first_block = offset >> blkbits; unsigned long max_blocks = (length + (1 << blkbits) - 1) >> blkbits; - struct ext2_sb_info *sbi = EXT2_SB(inode->i_sb); bool new = false, boundary = false; u32 bno; int ret; @@ -841,10 +818,7 @@ static int ext2_iomap_begin(struct inode *inode, loff_t offset, loff_t length, iomap->flags = 0; iomap->offset = (u64)first_block << blkbits; - if (flags & IOMAP_DAX) - iomap->dax_dev = sbi->s_daxdev; - else - iomap->bdev = inode->i_sb->s_bdev; + iomap->bdev = inode->i_sb->s_bdev; if (ret == 0) { /* @@ -859,8 +833,6 @@ static int ext2_iomap_begin(struct inode *inode, loff_t offset, loff_t length, } else { iomap->type = IOMAP_MAPPED; iomap->addr = (u64)bno << blkbits; - if (flags & IOMAP_DAX) - iomap->addr += sbi->s_dax_part_off; iomap->length = (u64)ret << blkbits; iomap->flags |= IOMAP_F_MERGED; } @@ -962,13 +934,6 @@ ext2_writepages(struct address_space *mapping, struct writeback_control *wbc) return mpage_writepages(mapping, wbc, ext2_get_block); } -static int -ext2_dax_writepages(struct address_space *mapping, struct writeback_control *wbc) -{ - struct ext2_sb_info *sbi = EXT2_SB(mapping->host->i_sb); - - return dax_writeback_mapping_range(mapping, sbi->s_daxdev, wbc); -} const struct address_space_operations ext2_aops = { .dirty_folio = block_dirty_folio, @@ -984,10 +949,6 @@ const struct address_space_operations ext2_aops = { .error_remove_folio = generic_error_remove_folio, }; -static const struct address_space_operations ext2_dax_aops = { - .writepages = ext2_dax_writepages, - .dirty_folio = noop_dirty_folio, -}; /* * Probably it should be a library function... search for first non-zero word @@ -1186,9 +1147,6 @@ static void __ext2_truncate_blocks(struct inode *inode, loff_t offset) blocksize = inode->i_sb->s_blocksize; iblock = (offset + blocksize-1) >> EXT2_BLOCK_SIZE_BITS(inode->i_sb); -#ifdef CONFIG_FS_DAX - WARN_ON(!rwsem_is_locked(&inode->i_mapping->invalidate_lock)); -#endif n = ext2_block_to_path(inode, iblock, offsets, NULL); if (n == 0) @@ -1290,12 +1248,7 @@ static int ext2_setsize(struct inode *inode, loff_t newsize) inode_dio_wait(inode); - if (IS_DAX(inode)) - error = dax_truncate_page(inode, newsize, NULL, - &ext2_iomap_ops); - else - error = block_truncate_page(inode->i_mapping, - newsize, ext2_get_block); + error = block_truncate_page(inode->i_mapping, newsize, ext2_get_block); if (error) return error; @@ -1363,7 +1316,7 @@ void ext2_set_inode_flags(struct inode *inode) unsigned int flags = EXT2_I(inode)->i_flags; inode->i_flags &= ~(S_SYNC | S_APPEND | S_IMMUTABLE | S_NOATIME | - S_DIRSYNC | S_DAX); + S_DIRSYNC); if (flags & EXT2_SYNC_FL) inode->i_flags |= S_SYNC; if (flags & EXT2_APPEND_FL) @@ -1374,18 +1327,13 @@ void ext2_set_inode_flags(struct inode *inode) inode->i_flags |= S_NOATIME; if (flags & EXT2_DIRSYNC_FL) inode->i_flags |= S_DIRSYNC; - if (test_opt(inode->i_sb, DAX) && S_ISREG(inode->i_mode)) - inode->i_flags |= S_DAX; } void ext2_set_file_ops(struct inode *inode) { inode->i_op = &ext2_file_inode_operations; inode->i_fop = &ext2_file_operations; - if (IS_DAX(inode)) - inode->i_mapping->a_ops = &ext2_dax_aops; - else - inode->i_mapping->a_ops = &ext2_aops; + inode->i_mapping->a_ops = &ext2_aops; } struct inode *ext2_iget (struct super_block *sb, unsigned long ino) diff --git a/fs/ext2/super.c b/fs/ext2/super.c index 19f76d8cb473..3999f8f3b156 100644 --- a/fs/ext2/super.c +++ b/fs/ext2/super.c @@ -34,7 +34,6 @@ #include #include #include -#include #include #include "ext2.h" #include "xattr.h" @@ -198,7 +197,6 @@ static void ext2_put_super (struct super_block * sb) brelse (sbi->s_sbh); sb->s_fs_info = NULL; kfree(sbi->s_blockgroup_lock); - fs_put_dax(sbi->s_daxdev, NULL); kfree(sbi); } @@ -329,11 +327,7 @@ static int ext2_show_options(struct seq_file *seq, struct dentry *root) if (test_opt(sb, GRPQUOTA)) seq_puts(seq, ",grpquota"); - if (test_opt(sb, XIP)) - seq_puts(seq, ",xip"); - if (test_opt(sb, DAX)) - seq_puts(seq, ",dax"); if (!test_opt(sb, RESERVATION)) seq_puts(seq, ",noreservation"); @@ -596,18 +590,10 @@ static int ext2_parse_param(struct fs_context *fc, struct fs_parameter *param) break; #endif case Opt_xip: - ext2_msg_fc(fc, KERN_INFO, "use dax instead of xip"); - ctx_set_mount_opt(ctx, EXT2_MOUNT_XIP); - fallthrough; + ext2_msg_fc(fc, KERN_WARNING, "DAX support has been removed. xip option ignored."); + break; case Opt_dax: -#ifdef CONFIG_FS_DAX - ext2_msg_fc(fc, KERN_WARNING, - "DAX enabled. Warning: DAX support in ext2 driver is deprecated" - " and will be removed at the end of 2025. Please use ext4 driver instead."); - ctx_set_mount_opt(ctx, EXT2_MOUNT_DAX); -#else - ext2_msg_fc(fc, KERN_INFO, "dax option not supported"); -#endif + ext2_msg_fc(fc, KERN_WARNING, "DAX support has been removed. dax option ignored."); break; #if defined(CONFIG_QUOTA) @@ -906,8 +892,6 @@ static int ext2_fill_super(struct super_block *sb, struct fs_context *fc) } sb->s_fs_info = sbi; sbi->s_sb_block = sb_block; - sbi->s_daxdev = fs_dax_get_by_bdev(sb->s_bdev, &sbi->s_dax_part_off, - NULL, NULL); spin_lock_init(&sbi->s_lock); ret = -EINVAL; @@ -992,17 +976,6 @@ static int ext2_fill_super(struct super_block *sb, struct fs_context *fc) } blocksize = BLOCK_SIZE << le32_to_cpu(sbi->s_es->s_log_block_size); - if (test_opt(sb, DAX)) { - if (!sbi->s_daxdev) { - ext2_msg(sb, KERN_ERR, - "DAX unsupported by block device. Turning off DAX."); - clear_opt(sbi->s_mount_opt, DAX); - } else if (blocksize != PAGE_SIZE) { - ext2_msg(sb, KERN_ERR, "unsupported blocksize for DAX\n"); - clear_opt(sbi->s_mount_opt, DAX); - } - } - /* If the blocksize doesn't match, re-read the thing.. */ if (sb->s_blocksize != blocksize) { brelse(bh); @@ -1252,7 +1225,6 @@ failed_mount_group_desc: failed_mount: brelse(bh); failed_sbi: - fs_put_dax(sbi->s_daxdev, NULL); sb->s_fs_info = NULL; kfree(sbi->s_blockgroup_lock); kfree(sbi); @@ -1379,11 +1351,6 @@ static int ext2_reconfigure(struct fs_context *fc) spin_lock(&sbi->s_lock); es = sbi->s_es; - if ((sbi->s_mount_opt ^ new_opts.s_mount_opt) & EXT2_MOUNT_DAX) { - ext2_msg(sb, KERN_WARNING, "warning: refusing change of " - "dax flag with busy inodes while remounting"); - new_opts.s_mount_opt ^= EXT2_MOUNT_DAX; - } if ((bool)(flags & SB_RDONLY) == sb_rdonly(sb)) goto out_set; if (flags & SB_RDONLY) { diff --git a/fs/hfs/bnode.c b/fs/hfs/bnode.c index 13d58c51fc46..da8e5342c91c 100644 --- a/fs/hfs/bnode.c +++ b/fs/hfs/bnode.c @@ -41,7 +41,7 @@ u32 check_and_correct_requested_length(struct hfs_bnode *node, u32 off, u32 len) node_size = node->tree->node_size; - if ((off + len) > node_size) { + if ((u64)off + len > node_size) { u32 new_len = node_size - off; pr_err("requested length has been corrected: " @@ -64,6 +64,8 @@ void hfs_bnode_read(struct hfs_bnode *node, void *buf, u32 off, u32 len) u32 bytes_read; u32 bytes_to_read; + memset(buf, 0, len); + if (!is_bnode_offset_valid(node, off)) return; @@ -344,7 +346,7 @@ static struct hfs_bnode *__hfs_bnode_create(struct hfs_btree *tree, u32 cnid) struct hfs_bnode *node, *node2; struct address_space *mapping; struct page *page; - int size, block, i, hash; + int block, i, hash; loff_t off; if (cnid >= tree->node_count) { @@ -352,9 +354,7 @@ static struct hfs_bnode *__hfs_bnode_create(struct hfs_btree *tree, u32 cnid) return NULL; } - size = sizeof(struct hfs_bnode) + tree->pages_per_bnode * - sizeof(struct page *); - node = kzalloc(size, GFP_KERNEL); + node = kzalloc_flex(*node, page, tree->pages_per_bnode, GFP_KERNEL); if (!node) return NULL; node->tree = tree; diff --git a/fs/hfs/catalog.c b/fs/hfs/catalog.c index 7f5339ee57c1..1bfa36d71e24 100644 --- a/fs/hfs/catalog.c +++ b/fs/hfs/catalog.c @@ -340,7 +340,6 @@ int hfs_cat_delete(u32 cnid, struct inode *dir, const struct qstr *str) { struct super_block *sb; struct hfs_find_data fd; - struct hfs_readdir_data *rd; int res, type; hfs_dbg("name %s, cnid %u\n", str ? str->name : NULL, cnid); @@ -366,14 +365,6 @@ int hfs_cat_delete(u32 cnid, struct inode *dir, const struct qstr *str) } } - /* we only need to take spinlock for exclusion with ->release() */ - spin_lock(&HFS_I(dir)->open_dir_lock); - list_for_each_entry(rd, &HFS_I(dir)->open_dir_list, list) { - if (fd.tree->keycmp(fd.search_key, (void *)&rd->key) < 0) - rd->file->f_pos--; - } - spin_unlock(&HFS_I(dir)->open_dir_lock); - res = hfs_brec_remove(&fd); if (res) goto out; diff --git a/fs/hfs/dir.c b/fs/hfs/dir.c index c4c6e1623f55..e13450bb933e 100644 --- a/fs/hfs/dir.c +++ b/fs/hfs/dir.c @@ -97,7 +97,15 @@ static int hfs_readdir(struct file *file, struct dir_context *ctx) } if (ctx->pos >= inode->i_size) goto out; - err = hfs_brec_goto(&fd, ctx->pos - 1); + rd = file->private_data; + if (rd && rd->pos == ctx->pos) { + memcpy(fd.search_key, &rd->key, sizeof(struct hfs_cat_key)); + err = hfs_brec_find(&fd); + if (err == -ENOENT) + err = hfs_brec_goto(&fd, 1); + } else { + err = hfs_brec_goto(&fd, ctx->pos - 1); + } if (err) goto out; @@ -146,7 +154,6 @@ static int hfs_readdir(struct file *file, struct dir_context *ctx) if (err) goto out; } - rd = file->private_data; if (!rd) { rd = kmalloc_obj(struct hfs_readdir_data); if (!rd) { @@ -154,15 +161,8 @@ static int hfs_readdir(struct file *file, struct dir_context *ctx) goto out; } file->private_data = rd; - rd->file = file; - spin_lock(&HFS_I(inode)->open_dir_lock); - list_add(&rd->list, &HFS_I(inode)->open_dir_list); - spin_unlock(&HFS_I(inode)->open_dir_lock); } - /* - * Can be done after the list insertion; exclusion with - * hfs_delete_cat() is provided by directory lock. - */ + rd->pos = ctx->pos; memcpy(&rd->key, &fd.key->cat, sizeof(struct hfs_cat_key)); out: hfs_find_exit(&fd); @@ -171,13 +171,7 @@ out: static int hfs_dir_release(struct inode *inode, struct file *file) { - struct hfs_readdir_data *rd = file->private_data; - if (rd) { - spin_lock(&HFS_I(inode)->open_dir_lock); - list_del(&rd->list); - spin_unlock(&HFS_I(inode)->open_dir_lock); - kfree(rd); - } + kfree(file->private_data); return 0; } @@ -306,10 +300,15 @@ static int hfs_rename(struct mnt_idmap *idmap, struct inode *old_dir, res = hfs_cat_move(d_inode(old_dentry)->i_ino, old_dir, &old_dentry->d_name, new_dir, &new_dentry->d_name); - if (!res) + if (!res) { + struct inode *inode = d_inode(old_dentry); + hfs_cat_build_key(old_dir->i_sb, - (btree_key *)&HFS_I(d_inode(old_dentry))->cat_key, + (btree_key *)&HFS_I(inode)->cat_key, new_dir->i_ino, &new_dentry->d_name); + inode_set_ctime_current(inode); + mark_inode_dirty(inode); + } return res; } diff --git a/fs/hfs/hfs.h b/fs/hfs/hfs.h index 3f2293ff6fdd..2e958c46bc0a 100644 --- a/fs/hfs/hfs.h +++ b/fs/hfs/hfs.h @@ -14,8 +14,7 @@ /*======== Data structures kept in memory ========*/ struct hfs_readdir_data { - struct list_head list; - struct file *file; + loff_t pos; struct hfs_cat_key key; }; diff --git a/fs/hfs/hfs_fs.h b/fs/hfs/hfs_fs.h index 1b23448c9a48..f3624514fcb0 100644 --- a/fs/hfs/hfs_fs.h +++ b/fs/hfs/hfs_fs.h @@ -36,8 +36,6 @@ struct hfs_inode_info { struct hfs_cat_key cat_key; - struct list_head open_dir_list; - spinlock_t open_dir_lock; struct inode *rsrc_inode; struct mutex extents_lock; diff --git a/fs/hfs/inode.c b/fs/hfs/inode.c index f41cc261684d..ac4a9055c5c0 100644 --- a/fs/hfs/inode.c +++ b/fs/hfs/inode.c @@ -196,8 +196,6 @@ struct inode *hfs_new_inode(struct inode *dir, const struct qstr *name, umode_t err = -ERANGE; mutex_init(&HFS_I(inode)->extents_lock); - INIT_LIST_HEAD(&HFS_I(inode)->open_dir_list); - spin_lock_init(&HFS_I(inode)->open_dir_lock); hfs_cat_build_key(sb, (btree_key *)&HFS_I(inode)->cat_key, dir->i_ino, name); next_id = atomic64_inc_return(&HFS_SB(sb)->next_id); if (next_id > U32_MAX) { @@ -205,7 +203,7 @@ struct inode *hfs_new_inode(struct inode *dir, const struct qstr *name, umode_t pr_err("cannot create new inode: next CNID exceeds limit\n"); goto out_discard; } - inode->i_ino = (u32)next_id; + inode->i_ino = (u32)next_id - 1; inode->i_mode = mode; inode->i_uid = current_fsuid(); inode->i_gid = current_fsgid(); @@ -349,12 +347,11 @@ static int hfs_read_inode(struct inode *inode, void *data) struct hfs_iget_data *idata = data; struct hfs_sb_info *hsb = HFS_SB(inode->i_sb); hfs_cat_rec *rec; + struct timespec64 mtime; HFS_I(inode)->flags = 0; HFS_I(inode)->rsrc_inode = NULL; mutex_init(&HFS_I(inode)->extents_lock); - INIT_LIST_HEAD(&HFS_I(inode)->open_dir_list); - spin_lock_init(&HFS_I(inode)->open_dir_lock); /* Initialize the inode */ inode->i_uid = hsb->s_uid; @@ -384,8 +381,10 @@ static int hfs_read_inode(struct inode *inode, void *data) inode->i_mode |= S_IWUGO; inode->i_mode &= ~hsb->s_file_umask; inode->i_mode |= S_IFREG; - inode_set_mtime_to_ts(inode, - inode_set_atime_to_ts(inode, inode_set_ctime_to_ts(inode, hfs_m_to_utime(rec->file.MdDat)))); + mtime = hfs_m_to_utime(rec->file.MdDat); + inode_set_ctime_to_ts(inode, mtime); + inode_set_atime_to_ts(inode, mtime); + inode_set_mtime_to_ts(inode, mtime); inode->i_op = &hfs_file_inode_operations; inode->i_fop = &hfs_file_operations; inode->i_mapping->a_ops = &hfs_aops; @@ -395,8 +394,10 @@ static int hfs_read_inode(struct inode *inode, void *data) inode->i_size = be16_to_cpu(rec->dir.Val) + 2; HFS_I(inode)->fs_blocks = 0; inode->i_mode = S_IFDIR | (S_IRWXUGO & ~hsb->s_dir_umask); - inode_set_mtime_to_ts(inode, - inode_set_atime_to_ts(inode, inode_set_ctime_to_ts(inode, hfs_m_to_utime(rec->dir.MdDat)))); + mtime = hfs_m_to_utime(rec->dir.MdDat); + inode_set_ctime_to_ts(inode, mtime); + inode_set_atime_to_ts(inode, mtime); + inode_set_mtime_to_ts(inode, mtime); inode->i_op = &hfs_dir_inode_operations; inode->i_fop = &hfs_dir_operations; break; @@ -666,7 +667,7 @@ int hfs_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, truncate_setsize(inode, attr->ia_size); hfs_file_truncate(inode); - simple_inode_init_ts(inode); + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); } setattr_copy(&nop_mnt_idmap, inode, attr); diff --git a/fs/hfs/super.c b/fs/hfs/super.c index a4f2a2bfa6d3..a466c401f6bb 100644 --- a/fs/hfs/super.c +++ b/fs/hfs/super.c @@ -338,7 +338,7 @@ static int hfs_fill_super(struct super_block *sb, struct fs_context *fc) sbi->sb = sb; sb->s_op = &hfs_super_operations; sb->s_xattr = hfs_xattr_handlers; - sb->s_flags |= SB_NODIRATIME; + sb->s_flags |= SB_NOATIME | SB_NODIRATIME; mutex_init(&sbi->bitmap_lock); res = hfs_mdb_get(sb); diff --git a/fs/hfsplus/bnode.c b/fs/hfsplus/bnode.c index f8b5a8ae58ff..d088fb7eb0df 100644 --- a/fs/hfsplus/bnode.c +++ b/fs/hfsplus/bnode.c @@ -25,6 +25,8 @@ void hfs_bnode_read(struct hfs_bnode *node, void *buf, u32 off, u32 len) struct page **pagep; u32 l; + memset(buf, 0, len); + if (!is_bnode_offset_valid(node, off)) return; @@ -455,7 +457,7 @@ static struct hfs_bnode *__hfs_bnode_create(struct hfs_btree *tree, u32 cnid) struct hfs_bnode *node, *node2; struct address_space *mapping; struct page *page; - int size, block, i, hash; + int block, i, hash; loff_t off; if (cnid >= tree->node_count) { @@ -464,9 +466,7 @@ static struct hfs_bnode *__hfs_bnode_create(struct hfs_btree *tree, u32 cnid) return NULL; } - size = sizeof(struct hfs_bnode) + tree->pages_per_bnode * - sizeof(struct page *); - node = kzalloc(size, GFP_KERNEL); + node = kzalloc_flex(*node, page, tree->pages_per_bnode); if (!node) return NULL; node->tree = tree; diff --git a/fs/hfsplus/btree.c b/fs/hfsplus/btree.c index 761c74ccd653..394542a47e60 100644 --- a/fs/hfsplus/btree.c +++ b/fs/hfsplus/btree.c @@ -365,6 +365,8 @@ struct hfs_btree *hfs_btree_open(struct super_block *sb, u32 id) } size = tree->node_size; + if (size < HFSPLUS_NODE_MINSZ || size > HFSPLUS_NODE_MXSZ) + goto fail_page; if (!is_power_of_2(size)) goto fail_page; if (!tree->node_count) diff --git a/fs/hfsplus/catalog.c b/fs/hfsplus/catalog.c index 9bc8a6c48fd3..776ce36cf076 100644 --- a/fs/hfsplus/catalog.c +++ b/fs/hfsplus/catalog.c @@ -333,7 +333,6 @@ int hfsplus_delete_cat(u32 cnid, struct inode *dir, const struct qstr *str) struct super_block *sb = dir->i_sb; struct hfs_find_data fd; struct hfsplus_fork_raw fork; - struct list_head *pos; int err, off; u16 type; @@ -392,16 +391,6 @@ int hfsplus_delete_cat(u32 cnid, struct inode *dir, const struct qstr *str) hfsplus_free_fork(sb, cnid, &fork, HFSPLUS_TYPE_RSRC); } - /* we only need to take spinlock for exclusion with ->release() */ - spin_lock(&HFSPLUS_I(dir)->open_dir_lock); - list_for_each(pos, &HFSPLUS_I(dir)->open_dir_list) { - struct hfsplus_readdir_data *rd = - list_entry(pos, struct hfsplus_readdir_data, list); - if (fd.tree->keycmp(fd.search_key, (void *)&rd->key) < 0) - rd->file->f_pos--; - } - spin_unlock(&HFSPLUS_I(dir)->open_dir_lock); - err = hfs_brec_remove(&fd); if (err) goto out; diff --git a/fs/hfsplus/dir.c b/fs/hfsplus/dir.c index 47194370c2c5..8bf6c7cdd9a8 100644 --- a/fs/hfsplus/dir.c +++ b/fs/hfsplus/dir.c @@ -185,7 +185,15 @@ static int hfsplus_readdir(struct file *file, struct dir_context *ctx) } if (ctx->pos >= inode->i_size) goto out; - err = hfs_brec_goto(&fd, ctx->pos - 1); + rd = file->private_data; + if (rd && rd->pos == ctx->pos) { + memcpy(fd.search_key, &rd->key, sizeof(struct hfsplus_cat_key)); + err = hfs_brec_find(&fd, hfs_find_rec_by_key); + if (err == -ENOENT) + err = hfs_brec_goto(&fd, 1); + } else { + err = hfs_brec_goto(&fd, ctx->pos - 1); + } if (err) goto out; for (;;) { @@ -261,7 +269,6 @@ next: if (err) goto out; } - rd = file->private_data; if (!rd) { rd = kmalloc_obj(struct hfsplus_readdir_data); if (!rd) { @@ -269,15 +276,8 @@ next: goto out; } file->private_data = rd; - rd->file = file; - spin_lock(&HFSPLUS_I(inode)->open_dir_lock); - list_add(&rd->list, &HFSPLUS_I(inode)->open_dir_list); - spin_unlock(&HFSPLUS_I(inode)->open_dir_lock); } - /* - * Can be done after the list insertion; exclusion with - * hfsplus_delete_cat() is provided by directory lock. - */ + rd->pos = ctx->pos; memcpy(&rd->key, fd.key, sizeof(struct hfsplus_cat_key)); out: kfree(strbuf); @@ -287,13 +287,7 @@ out: static int hfsplus_dir_release(struct inode *inode, struct file *file) { - struct hfsplus_readdir_data *rd = file->private_data; - if (rd) { - spin_lock(&HFSPLUS_I(inode)->open_dir_lock); - list_del(&rd->list); - spin_unlock(&HFSPLUS_I(inode)->open_dir_lock); - kfree(rd); - } + kfree(file->private_data); return 0; } diff --git a/fs/hfsplus/hfsplus_fs.h b/fs/hfsplus/hfsplus_fs.h index 3545b8dbf11c..ec04b82ad927 100644 --- a/fs/hfsplus/hfsplus_fs.h +++ b/fs/hfsplus/hfsplus_fs.h @@ -214,8 +214,6 @@ struct hfsplus_inode_info { sector_t fs_blocks; u8 userflags; /* BSD user file flags */ u32 subfolders; /* Subfolder count (HFSX only) */ - struct list_head open_dir_list; - spinlock_t open_dir_lock; loff_t phys_size; struct inode vfs_inode; @@ -272,8 +270,7 @@ struct hfs_find_data { }; struct hfsplus_readdir_data { - struct list_head list; - struct file *file; + loff_t pos; struct hfsplus_cat_key key; }; @@ -600,7 +597,7 @@ u32 check_and_correct_requested_length(struct hfs_bnode *node, u32 off, u32 len) node_size = node->tree->node_size; - if ((off + len) > node_size) { + if ((u64)off + len > node_size) { u32 new_len = node_size - off; pr_err("requested length has been corrected: " diff --git a/fs/hfsplus/inode.c b/fs/hfsplus/inode.c index 5565c14b4bf6..1c57c873f05d 100644 --- a/fs/hfsplus/inode.c +++ b/fs/hfsplus/inode.c @@ -125,9 +125,44 @@ static ssize_t hfsplus_direct_IO(struct kiocb *iocb, struct iov_iter *iter) struct file *file = iocb->ki_filp; struct address_space *mapping = file->f_mapping; struct inode *inode = mapping->host; + loff_t isize; size_t count = iov_iter_count(iter); + loff_t end = iocb->ki_pos + count; ssize_t ret; + /* + * The hfsplus_get_block() only allows creating the next sequential block. + * For direct writes beyond EOF, expand the file first. + */ + if (iov_iter_rw(iter) == WRITE && iocb->ki_pos > i_size_read(inode)) { + loff_t start_off, end_off; + loff_t start_page, end_page; + + isize = i_size_read(inode); + + /* + * Wait for any in-flight DIO on this inode to finish before + * calling generic_cont_expand_simple(). + */ + inode_dio_wait(inode); + + ret = generic_cont_expand_simple(inode, iocb->ki_pos); + if (ret) + return ret; + + start_off = isize; + end_off = (end > 0) ? end - 1 : end; + + ret = filemap_write_and_wait_range(mapping, start_off, end_off); + if (ret) + return ret; + + start_page = start_off >> PAGE_SHIFT; + end_page = end_off >> PAGE_SHIFT; + + invalidate_inode_pages2_range(mapping, start_page, end_page); + } + ret = blockdev_direct_IO(iocb, inode, iter, hfsplus_get_block); /* @@ -135,8 +170,7 @@ static ssize_t hfsplus_direct_IO(struct kiocb *iocb, struct iov_iter *iter) * blocks outside i_size. Trim these off again. */ if (unlikely(iov_iter_rw(iter) == WRITE && ret < 0)) { - loff_t isize = i_size_read(inode); - loff_t end = iocb->ki_pos + count; + isize = i_size_read(inode); if (end > isize) hfsplus_write_failed(mapping, end); @@ -449,8 +483,6 @@ struct inode *hfsplus_new_inode(struct super_block *sb, struct inode *dir, simple_inode_init_ts(inode); hip = HFSPLUS_I(inode); - INIT_LIST_HEAD(&hip->open_dir_list); - spin_lock_init(&hip->open_dir_lock); mutex_init(&hip->extents_lock); atomic_set(&hip->opencnt, 0); hip->extent_state = 0; diff --git a/fs/hfsplus/super.c b/fs/hfsplus/super.c index 40a0feda716b..5777e31de45a 100644 --- a/fs/hfsplus/super.c +++ b/fs/hfsplus/super.c @@ -91,8 +91,6 @@ struct inode *hfsplus_iget(struct super_block *sb, unsigned long ino) HFSPLUS_I(inode)->fs_blocks = 0; HFSPLUS_I(inode)->userflags = 0; HFSPLUS_I(inode)->subfolders = 0; - INIT_LIST_HEAD(&HFSPLUS_I(inode)->open_dir_list); - spin_lock_init(&HFSPLUS_I(inode)->open_dir_lock); HFSPLUS_I(inode)->phys_size = 0; if (inode->i_ino >= HFSPLUS_FIRSTUSER_CNID || diff --git a/fs/hfsplus/xattr.c b/fs/hfsplus/xattr.c index 452a1f9becb2..21a1c196c71f 100644 --- a/fs/hfsplus/xattr.c +++ b/fs/hfsplus/xattr.c @@ -317,7 +317,6 @@ check_attr_tree_state_again: next_node++; } - hfsplus_mark_inode_dirty(HFSPLUS_ATTR_TREE_I(sb), HFSPLUS_I_ATTR_DIRTY); hfsplus_mark_inode_dirty(attr_file, HFSPLUS_I_ATTR_DIRTY); sbi->attr_tree = hfs_btree_open(sb, HFSPLUS_ATTR_CNID); diff --git a/fs/isofs/rock.c b/fs/isofs/rock.c index 1232fab59a4e..2628f31bd3a5 100644 --- a/fs/isofs/rock.c +++ b/fs/isofs/rock.c @@ -466,6 +466,9 @@ repeat: inode->i_size = symlink_len; while (slen > 1) { rootflag = 0; + /* keep the component within the SL record */ + if (slp->len + 2 > slen) + goto eio; switch (slp->flags & ~1) { case 0: inode->i_size += @@ -621,6 +624,14 @@ static char *get_symlink_chunk(char *rpnt, struct rock_ridge *rr, char *plimit) slp = &rr->u.SL.link; while (slen > 1) { rootflag = 0; + /* + * A component is slp->len + 2 bytes (a two-byte header plus + * len bytes of text). If it does not fit in the bytes left in + * the SL record the record is malformed: fail like the plimit + * checks below so readlink() returns -EIO, not a truncated path. + */ + if (slp->len + 2 > slen) + return NULL; switch (slp->flags & ~1) { case 0: if (slp->len > plimit - rpnt) diff --git a/fs/nilfs2/namei.c b/fs/nilfs2/namei.c index 40ac679ec56e..e2fe95de3d71 100644 --- a/fs/nilfs2/namei.c +++ b/fs/nilfs2/namei.c @@ -258,7 +258,7 @@ out: else nilfs_transaction_abort(dir->i_sb); - return ERR_PTR(err); + return err ? ERR_PTR(err) : NULL; out_fail: drop_nlink(inode); diff --git a/fs/nilfs2/segment.c b/fs/nilfs2/segment.c index 1491a4d4b1e1..9332f5ac6083 100644 --- a/fs/nilfs2/segment.c +++ b/fs/nilfs2/segment.c @@ -2512,12 +2512,33 @@ int nilfs_clean_segments(struct super_block *sb, struct nilfs_argv *argv, struct nilfs_sc_info *sci = nilfs->ns_writer; struct nilfs_transaction_info ti; int err; + size_t i, nfreesegs = argv[4].v_nmembs; + __u64 *segnumv = kbufs[4]; if (unlikely(!sci)) return -EROFS; nilfs_transaction_lock(sb, &ti, 1); + /* + * Validate segment numbers under ns_segctor_sem (held for write + * by nilfs_transaction_lock above) so the check is serialized + * against nilfs_ioctl_resize(), which can modify ns_nsegments. + * Rejecting bad input here, before any segment-cleaning work + * begins, avoids the per-element diagnostic path inside + * nilfs_sufile_updatev() that would otherwise run under this + * same lock and stall concurrent readers. + */ + for (i = 0; i < nfreesegs; i++) { + if (segnumv[i] >= nilfs->ns_nsegments) { + nilfs_err(sb, + "Segment number %llu to be freed is out of range", + (unsigned long long)segnumv[i]); + err = -EINVAL; + goto bail_unlock; + } + } + err = nilfs_mdt_save_to_shadow_map(nilfs->ns_dat); if (unlikely(err)) goto out_unlock; @@ -2558,6 +2579,7 @@ int nilfs_clean_segments(struct super_block *sb, struct nilfs_argv *argv, sci->sc_freesegs = NULL; sci->sc_nfreesegs = 0; nilfs_mdt_clear_shadow_map(nilfs->ns_dat); + bail_unlock: nilfs_transaction_unlock(sb); return err; } diff --git a/fs/nilfs2/super.c b/fs/nilfs2/super.c index 7aa5ef8606cd..893a504cb80c 100644 --- a/fs/nilfs2/super.c +++ b/fs/nilfs2/super.c @@ -1070,8 +1070,6 @@ nilfs_fill_super(struct super_block *sb, struct fs_context *fc) sb->s_time_gran = 1; sb->s_max_links = NILFS_LINK_MAX; - sb->s_bdi = bdi_get(sb->s_bdev->bd_disk->bdi); - err = load_nilfs(nilfs, sb); if (err) goto failed_nilfs; diff --git a/fs/notify/fanotify/fanotify.c b/fs/notify/fanotify/fanotify.c index 38290b9c07f7..a3555bebad63 100644 --- a/fs/notify/fanotify/fanotify.c +++ b/fs/notify/fanotify/fanotify.c @@ -14,6 +14,7 @@ #include #include #include +#include #include "fanotify.h" @@ -120,10 +121,7 @@ static bool fanotify_error_event_equal(struct fanotify_error_event *fee1, struct fanotify_error_event *fee2) { /* Error events against the same file system are always merged. */ - if (!fanotify_fsid_equal(&fee1->fsid, &fee2->fsid)) - return false; - - return true; + return fanotify_fsid_equal(&fee1->fsid, &fee2->fsid); } static bool fanotify_should_merge(struct fanotify_event *old, @@ -842,6 +840,15 @@ static struct fanotify_event *fanotify_alloc_event( /* Whoever is interested in the event, pays for the allocation. */ old_memcg = set_active_memcg(group->memcg); + if (FAN_GROUP_FLAG(group, FAN_REPORT_TID)) + pid = task_pid(current); + else + pid = task_tgid(current); + + if (FAN_GROUP_FLAG(group, FAN_REPORT_PIDFD) && + pidfs_register_pid_gfp(pid, gfp)) + goto out; + if (fanotify_is_perm_event(mask)) { event = fanotify_alloc_perm_event(data, data_type, gfp); } else if (fanotify_is_error_event(mask)) { @@ -863,15 +870,10 @@ static struct fanotify_event *fanotify_alloc_event( if (!event) goto out; - if (FAN_GROUP_FLAG(group, FAN_REPORT_TID)) - pid = get_pid(task_pid(current)); - else - pid = get_pid(task_tgid(current)); - /* Mix event info, FAN_ONDIR flag and pid into event merge key */ hash ^= hash_long((unsigned long)pid | ondir, FANOTIFY_EVENT_HASH_BITS); fanotify_init_event(event, hash, mask); - event->pid = pid; + event->pid = get_pid(pid); out: set_active_memcg(old_memcg); diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c index ae904451dfc0..b604e3da58ad 100644 --- a/fs/notify/fanotify/fanotify_user.c +++ b/fs/notify/fanotify/fanotify_user.c @@ -19,6 +19,7 @@ #include #include #include +#include #include @@ -903,25 +904,13 @@ static ssize_t copy_event_to_user(struct fsnotify_group *group, metadata.fd = fd >= 0 ? fd : FAN_NOFD; if (pidfd_mode) { - /* - * Complain if the FAN_REPORT_PIDFD and FAN_REPORT_TID mutual - * exclusion is ever lifted. At the time of incoporating pidfd - * support within fanotify, the pidfd API only supported the - * creation of pidfds for thread-group leaders. - */ - WARN_ON_ONCE(FAN_GROUP_FLAG(group, FAN_REPORT_TID)); + unsigned int pidfd_flags = PIDFD_STALE; - /* - * The PIDTYPE_TGID check for an event->pid is performed - * preemptively in an attempt to catch out cases where the event - * listener reads events after the event generating process has - * already terminated. Depending on flag FAN_REPORT_FD_ERROR, - * report either -ESRCH or FAN_NOPIDFD to the event listener in - * those cases with all other pidfd creation errors reported as - * the error code itself or as FAN_EPIDFD. - */ - if (metadata.pid && pid_has_task(event->pid, PIDTYPE_TGID)) - pidfd = pidfd_prepare(event->pid, 0, &pidfd_file); + if (FAN_GROUP_FLAG(group, FAN_REPORT_TID)) + pidfd_flags |= PIDFD_THREAD; + + if (metadata.pid) + pidfd = pidfd_prepare(event->pid, pidfd_flags, &pidfd_file); if (!FAN_GROUP_FLAG(group, FAN_REPORT_FD_ERROR) && pidfd < 0) pidfd = pidfd == -ESRCH ? FAN_NOPIDFD : FAN_EPIDFD; @@ -1628,14 +1617,6 @@ SYSCALL_DEFINE2(fanotify_init, unsigned int, flags, unsigned int, event_f_flags) #endif return -EINVAL; - /* - * A pidfd can only be returned for a thread-group leader; thus - * FAN_REPORT_PIDFD and FAN_REPORT_TID need to remain mutually - * exclusive. - */ - if ((flags & FAN_REPORT_PIDFD) && (flags & FAN_REPORT_TID)) - return -EINVAL; - /* Don't allow mixing mnt events with inode events for now */ if (flags & FAN_REPORT_MNT) { if (class != FAN_CLASS_NOTIF) diff --git a/fs/pidfs.c b/fs/pidfs.c index fdd75c3eaff8..aaa609ddab04 100644 --- a/fs/pidfs.c +++ b/fs/pidfs.c @@ -977,14 +977,16 @@ static void pidfs_put_data(void *data) } /** - * pidfs_register_pid - register a struct pid in pidfs + * pidfs_register_pid_gfp - register a struct pid in pidfs with custom GFP + * flags * @pid: pid to pin + * @gfp: GFP flags for memory allocation * - * Register a struct pid in pidfs. + * Register a struct pid in pidfs with custom GFP flags. * * Return: On success zero, on error a negative error code is returned. */ -int pidfs_register_pid(struct pid *pid) +int pidfs_register_pid_gfp(struct pid *pid, gfp_t gfp) { struct pidfs_attr *new_attr __free(kfree) = NULL; struct pidfs_attr *attr; @@ -1000,7 +1002,7 @@ int pidfs_register_pid(struct pid *pid) if (attr) return 0; - new_attr = kmem_cache_zalloc(pidfs_attr_cachep, GFP_KERNEL); + new_attr = kmem_cache_zalloc(pidfs_attr_cachep, gfp); if (!new_attr) return -ENOMEM; diff --git a/fs/udf/balloc.c b/fs/udf/balloc.c index 807c493ed0cd..cc6dc6e1d84d 100644 --- a/fs/udf/balloc.c +++ b/fs/udf/balloc.c @@ -82,8 +82,9 @@ static int load_block_bitmap(struct super_block *sb, int nr_groups = bitmap->s_nr_groups; if (block_group >= nr_groups) { - udf_debug("block_group (%u) > nr_groups (%d)\n", + udf_debug("block_group (%u) >= nr_groups (%d)\n", block_group, nr_groups); + return -EFSCORRUPTED; } if (bitmap->s_block_bitmap[block_group]) { @@ -662,7 +663,7 @@ void udf_free_blocks(struct super_block *sb, struct inode *inode, if (check_add_overflow(bloc->logicalBlockNum, offset, &blk) || check_add_overflow(blk, count, &blk) || - bloc->logicalBlockNum + count > map->s_partition_len) { + blk > map->s_partition_len) { udf_debug("Invalid request to free blocks: (%d, %u), off %u, " "len %u, partition len %u\n", partition, bloc->logicalBlockNum, offset, count, diff --git a/fs/udf/super.c b/fs/udf/super.c index be4085da2104..7b85f5a2b79f 100644 --- a/fs/udf/super.c +++ b/fs/udf/super.c @@ -1242,6 +1242,11 @@ static int udf_load_vat(struct super_block *sb, int p_index, int type1_index) if (map->s_partition_type == UDF_VIRTUAL_MAP15) { map->s_type_specific.s_virtual.s_start_offset = 0; + if (sbi->s_vat_inode->i_size < 36) { + udf_err(sb, "Too short VAT inode size %lld\n", + sbi->s_vat_inode->i_size); + return -EFSCORRUPTED; + } map->s_type_specific.s_virtual.s_num_entries = (sbi->s_vat_inode->i_size - 36) >> 2; } else if (map->s_partition_type == UDF_VIRTUAL_MAP20) { @@ -1263,6 +1268,14 @@ static int udf_load_vat(struct super_block *sb, int p_index, int type1_index) map->s_type_specific.s_virtual.s_start_offset = le16_to_cpu(vat20->lengthHeader); + if (map->s_type_specific.s_virtual.s_start_offset + > sbi->s_vat_inode->i_size) { + udf_err(sb, "Corrupted VAT header length %u (VAT inode size %lld)\n", + map->s_type_specific.s_virtual.s_start_offset, + sbi->s_vat_inode->i_size); + brelse(bh); + return -EFSCORRUPTED; + } map->s_type_specific.s_virtual.s_num_entries = (sbi->s_vat_inode->i_size - map->s_type_specific.s_virtual. @@ -1418,7 +1431,8 @@ static int udf_load_sparable_map(struct super_block *sb, if (ident != 0 || strncmp(st->sparingIdent.ident, UDF_ID_SPARING, strlen(UDF_ID_SPARING)) || - sizeof(*st) + le16_to_cpu(st->reallocationTableLen) > + struct_size(st, mapEntry, + le16_to_cpu(st->reallocationTableLen)) > sb->s_blocksize) { brelse(bh); continue; diff --git a/fs/xfs/libxfs/xfs_ag.c b/fs/xfs/libxfs/xfs_ag.c index dcd2f93b6a6c..6cab66781d66 100644 --- a/fs/xfs/libxfs/xfs_ag.c +++ b/fs/xfs/libxfs/xfs_ag.c @@ -863,32 +863,30 @@ resv_err: return err2; } -void -xfs_growfs_compute_deltas( +/* + * Return the agcount for the new file system size passed in *nb and adjust *nb + * when it has to be reduced because of maximum AG count or because it would + * create a below minimum size AG. + */ +xfs_agnumber_t +xfs_growfs_compute_agcount( struct xfs_mount *mp, - xfs_rfsblock_t nb, - int64_t *deltap, - xfs_agnumber_t *nagcountp) + xfs_rfsblock_t *nb) { - xfs_rfsblock_t nb_div, nb_mod; - int64_t delta; - xfs_agnumber_t nagcount; + uint64_t agcount; /* 64-bits wide to catch overflows */ + xfs_extlen_t remainder; - nb_div = nb; - nb_mod = do_div(nb_div, mp->m_sb.sb_agblocks); - if (nb_mod && nb_mod >= XFS_MIN_AG_BLOCKS) - nb_div++; - else if (nb_mod) - nb = nb_div * mp->m_sb.sb_agblocks; - - if (nb_div > XFS_MAX_AGNUMBER + 1) { - nb_div = XFS_MAX_AGNUMBER + 1; - nb = nb_div * mp->m_sb.sb_agblocks; + agcount = div_u64_rem(*nb, mp->m_sb.sb_agblocks, &remainder); + if (agcount >= XFS_MAX_AGNUMBER + 1) { + agcount = XFS_MAX_AGNUMBER + 1; + remainder = 0; } - nagcount = nb_div; - delta = nb - mp->m_sb.sb_dblocks; - *deltap = delta; - *nagcountp = nagcount; + *nb = (xfs_rfsblock_t)agcount * mp->m_sb.sb_agblocks; + if (remainder >= XFS_MIN_AG_BLOCKS) { + *nb += remainder; + agcount++; + } + return agcount; } /* diff --git a/fs/xfs/libxfs/xfs_ag.h b/fs/xfs/libxfs/xfs_ag.h index 16a9b43a3c27..fd22fe598931 100644 --- a/fs/xfs/libxfs/xfs_ag.h +++ b/fs/xfs/libxfs/xfs_ag.h @@ -329,12 +329,11 @@ struct aghdr_init_data { int xfs_ag_init_headers(struct xfs_mount *mp, struct aghdr_init_data *id); int xfs_ag_shrink_space(struct xfs_perag *pag, struct xfs_trans **tpp, xfs_extlen_t delta); -void -xfs_growfs_compute_deltas(struct xfs_mount *mp, xfs_rfsblock_t nb, - int64_t *deltap, xfs_agnumber_t *nagcountp); int xfs_ag_extend_space(struct xfs_perag *pag, struct xfs_trans *tp, xfs_extlen_t len); int xfs_ag_get_geometry(struct xfs_perag *pag, struct xfs_ag_geometry *ageo); +xfs_agnumber_t xfs_growfs_compute_agcount(struct xfs_mount *mp, + xfs_rfsblock_t *nb); static inline xfs_fsblock_t xfs_agbno_to_fsb( diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index 93caa1dae501..b3f7b2c34ad7 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -276,7 +276,7 @@ xfs_attr_get( return -EIO; if (!args->owner) - args->owner = args->dp->i_ino; + args->owner = I_INO(args->dp); args->geo = args->dp->i_mount->m_attr_geo; args->whichfork = XFS_ATTR_FORK; xfs_attr_sethash(args); diff --git a/fs/xfs/libxfs/xfs_attr_leaf.c b/fs/xfs/libxfs/xfs_attr_leaf.c index 2b78041e8672..86c5c09a5db4 100644 --- a/fs/xfs/libxfs/xfs_attr_leaf.c +++ b/fs/xfs/libxfs/xfs_attr_leaf.c @@ -1429,7 +1429,7 @@ xfs_attr3_leaf_init( struct xfs_da_args args = { .trans = tp, .dp = dp, - .owner = dp->i_ino, + .owner = I_INO(dp), .geo = dp->i_mount->m_attr_geo, }; diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index 7a4c8f1aa76c..d64defeda645 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -602,7 +602,7 @@ xfs_bmap_btree_to_extents( if ((error = xfs_btree_check_block(cur, cblock, 0, cbp))) return error; - xfs_rmap_ino_bmbt_owner(&oinfo, ip->i_ino, whichfork); + xfs_rmap_inode_bmbt_owner(&oinfo, ip, whichfork); error = xfs_free_extent_later(cur->bc_tp, cbno, 1, &oinfo, XFS_AG_RESV_NONE, 0); if (error) @@ -676,13 +676,12 @@ xfs_bmap_extents_to_btree( memset(&args, 0, sizeof(args)); args.tp = tp; args.mp = mp; - xfs_rmap_ino_bmbt_owner(&args.oinfo, ip->i_ino, whichfork); + xfs_rmap_inode_bmbt_owner(&args.oinfo, ip, whichfork); args.minlen = args.maxlen = args.prod = 1; args.wasdel = wasdel; *logflagsp = 0; - error = xfs_alloc_vextent_start_ag(&args, - XFS_INO_TO_FSB(mp, ip->i_ino)); + error = xfs_alloc_vextent_start_ag(&args, XFS_INODE_TO_FSB(ip)); if (error) goto out_root_realloc; @@ -820,7 +819,7 @@ xfs_bmap_local_to_extents( args.mp = ip->i_mount; args.total = total; args.minlen = args.maxlen = args.prod = 1; - xfs_rmap_ino_owner(&args.oinfo, ip->i_ino, whichfork, 0); + xfs_rmap_inode_owner(&args.oinfo, ip, whichfork, 0); /* * Allocate a block. We know we need only one, since the @@ -828,8 +827,7 @@ xfs_bmap_local_to_extents( */ args.total = total; args.minlen = args.maxlen = args.prod = 1; - error = xfs_alloc_vextent_start_ag(&args, - XFS_INO_TO_FSB(args.mp, ip->i_ino)); + error = xfs_alloc_vextent_start_ag(&args, XFS_INODE_TO_FSB(ip)); if (error) goto done; @@ -976,7 +974,7 @@ xfs_bmap_add_attrfork_local( dargs.total = dargs.geo->fsbcount; dargs.whichfork = XFS_DATA_FORK; dargs.trans = tp; - dargs.owner = ip->i_ino; + dargs.owner = I_INO(ip); return xfs_dir2_sf_to_block(&dargs); } @@ -1110,7 +1108,7 @@ xfs_bmap_complain_bad_rec( xfs_warn(mp, "Bmap BTree record corruption in inode 0x%llx %s fork detected at %pS!", - ip->i_ino, forkname, fa); + I_INO(ip), forkname, fa); xfs_warn(mp, "Offset 0x%llx, start block 0x%llx, block count 0x%llx state 0x%x", irec->br_startoff, irec->br_startblock, irec->br_blockcount, @@ -1143,7 +1141,7 @@ xfs_iread_bmbt_block( num_recs = xfs_btree_get_numrecs(block); if (unlikely(ir->loaded + num_recs > ifp->if_nextents)) { xfs_warn(ip->i_mount, "corrupt dinode %llu, (btree extents).", - (unsigned long long)ip->i_ino); + (unsigned long long)I_INO(ip)); xfs_inode_verifier_error(ip, -EFSCORRUPTED, __func__, block, sizeof(*block), __this_address); xfs_bmap_mark_sick(ip, whichfork); @@ -3590,7 +3588,7 @@ xfs_bmap_btalloc_best_length( xfs_extlen_t blen = 0; int error; - ap->blkno = XFS_INO_TO_FSB(args->mp, ap->ip->i_ino); + ap->blkno = XFS_INODE_TO_FSB(ap->ip); if (!xfs_bmap_adjacent(ap)) ap->eof = false; diff --git a/fs/xfs/libxfs/xfs_bmap_btree.c b/fs/xfs/libxfs/xfs_bmap_btree.c index 1c7165df483a..758a4b1ccf5b 100644 --- a/fs/xfs/libxfs/xfs_bmap_btree.c +++ b/fs/xfs/libxfs/xfs_bmap_btree.c @@ -36,10 +36,10 @@ xfs_bmbt_init_block( { if (bp) xfs_btree_init_buf(ip->i_mount, bp, &xfs_bmbt_ops, level, - numrecs, ip->i_ino); + numrecs, I_INO(ip)); else xfs_btree_init_block(ip->i_mount, buf, &xfs_bmbt_ops, level, - numrecs, ip->i_ino); + numrecs, I_INO(ip)); } /* @@ -217,7 +217,7 @@ xfs_bmbt_alloc_block( memset(&args, 0, sizeof(args)); args.tp = cur->bc_tp; args.mp = cur->bc_mp; - xfs_rmap_ino_bmbt_owner(&args.oinfo, cur->bc_ino.ip->i_ino, + xfs_rmap_inode_bmbt_owner(&args.oinfo, cur->bc_ino.ip, cur->bc_ino.whichfork); args.minlen = args.maxlen = args.prod = 1; args.wasdel = cur->bc_flags & XFS_BTREE_BMBT_WASDEL; @@ -280,7 +280,7 @@ xfs_bmbt_free_block( struct xfs_owner_info oinfo; int error; - xfs_rmap_ino_bmbt_owner(&oinfo, ip->i_ino, cur->bc_ino.whichfork); + xfs_rmap_inode_bmbt_owner(&oinfo, ip, cur->bc_ino.whichfork); error = xfs_free_extent_later(cur->bc_tp, fsbno, 1, &oinfo, XFS_AG_RESV_NONE, 0); if (error) diff --git a/fs/xfs/libxfs/xfs_bmap_btree.h b/fs/xfs/libxfs/xfs_bmap_btree.h index b238d559ab03..e0c870beaf67 100644 --- a/fs/xfs/libxfs/xfs_bmap_btree.h +++ b/fs/xfs/libxfs/xfs_bmap_btree.h @@ -89,7 +89,7 @@ xfs_bmbt_key_addr( { return (struct xfs_bmbt_key *) ((char *)block + xfs_bmbt_block_len(mp) + - (index - 1) * sizeof(struct xfs_bmbt_key *)); + (index - 1) * sizeof(struct xfs_bmbt_key)); } static inline xfs_bmbt_ptr_t * diff --git a/fs/xfs/libxfs/xfs_btree.c b/fs/xfs/libxfs/xfs_btree.c index 7012f3570c8d..60ef7f08b1d3 100644 --- a/fs/xfs/libxfs/xfs_btree.c +++ b/fs/xfs/libxfs/xfs_btree.c @@ -371,7 +371,7 @@ xfs_btree_check_ptr( case XFS_BTREE_TYPE_INODE: xfs_err(cur->bc_mp, "Inode %llu fork %d: Corrupt %sbt pointer at level %d index %d.", - cur->bc_ino.ip->i_ino, + I_INO(cur->bc_ino.ip), cur->bc_ino.whichfork, cur->bc_ops->name, level, index); break; @@ -1305,7 +1305,7 @@ xfs_btree_owner( case XFS_BTREE_TYPE_MEM: return cur->bc_mem.xfbtree->owner; case XFS_BTREE_TYPE_INODE: - return cur->bc_ino.ip->i_ino; + return I_INO(cur->bc_ino.ip); case XFS_BTREE_TYPE_AG: return cur->bc_group->xg_gno; default: @@ -3129,7 +3129,7 @@ xfs_btree_promote_leaf_iroot( */ broot = cur->bc_ops->broot_realloc(cur, 1); xfs_btree_init_block(cur->bc_mp, broot, cur->bc_ops, - cur->bc_nlevels - 1, 1, cur->bc_ino.ip->i_ino); + cur->bc_nlevels - 1, 1, I_INO(cur->bc_ino.ip)); pp = xfs_btree_ptr_addr(cur, 1, broot); kp = xfs_btree_key_addr(cur, 1, broot); @@ -3243,8 +3243,7 @@ xfs_btree_new_iroot( if (level > 0) aptr = *xfs_btree_ptr_addr(cur, 1, block); else - aptr.l = cpu_to_be64(XFS_INO_TO_FSB(cur->bc_mp, - cur->bc_ino.ip->i_ino)); + aptr.l = cpu_to_be64(XFS_INODE_TO_FSB(cur->bc_ino.ip)); /* Allocate the new block. If we can't do it, we're toast. Give up. */ error = xfs_btree_alloc_block(cur, &aptr, &nptr, stat); @@ -3827,7 +3826,7 @@ xfs_btree_demote_leaf_child( */ broot = cur->bc_ops->broot_realloc(cur, numrecs); xfs_btree_init_block(cur->bc_mp, broot, cur->bc_ops, 0, numrecs, - cur->bc_ino.ip->i_ino); + I_INO(cur->bc_ino.ip)); rp = xfs_btree_rec_addr(cur, 1, broot); crp = xfs_btree_rec_addr(cur, 1, cblock); @@ -5608,9 +5607,8 @@ xfs_btree_alloc_metafile_block( ASSERT(xfs_is_metadir_inode(ip)); - xfs_rmap_ino_bmbt_owner(&args.oinfo, ip->i_ino, cur->bc_ino.whichfork); - error = xfs_alloc_vextent_start_ag(&args, - XFS_INO_TO_FSB(cur->bc_mp, ip->i_ino)); + xfs_rmap_inode_bmbt_owner(&args.oinfo, ip, cur->bc_ino.whichfork); + error = xfs_alloc_vextent_start_ag(&args, XFS_INODE_TO_FSB(ip)); if (error) return error; if (args.fsbno == NULLFSBLOCK) { @@ -5641,7 +5639,7 @@ xfs_btree_free_metafile_block( ASSERT(xfs_is_metadir_inode(ip)); - xfs_rmap_ino_bmbt_owner(&oinfo, ip->i_ino, cur->bc_ino.whichfork); + xfs_rmap_inode_bmbt_owner(&oinfo, ip, cur->bc_ino.whichfork); error = xfs_free_extent_later(tp, fsbno, 1, &oinfo, XFS_AG_RESV_METAFILE, 0); if (error) diff --git a/fs/xfs/libxfs/xfs_btree_staging.c b/fs/xfs/libxfs/xfs_btree_staging.c index 4300c058807b..c3c7ea54895a 100644 --- a/fs/xfs/libxfs/xfs_btree_staging.c +++ b/fs/xfs/libxfs/xfs_btree_staging.c @@ -309,7 +309,7 @@ xfs_btree_bload_prep_block( /* Initialize it and send it out. */ xfs_btree_init_block(cur->bc_mp, ifp->if_broot, cur->bc_ops, - level, nr_this_block, cur->bc_ino.ip->i_ino); + level, nr_this_block, I_INO(cur->bc_ino.ip)); *bpp = NULL; *blockp = ifp->if_broot; diff --git a/fs/xfs/libxfs/xfs_da_btree.c b/fs/xfs/libxfs/xfs_da_btree.c index ad801b7bd2dd..9debb95d86fa 100644 --- a/fs/xfs/libxfs/xfs_da_btree.c +++ b/fs/xfs/libxfs/xfs_da_btree.c @@ -2780,7 +2780,7 @@ invalid_mapping: error = -EFSCORRUPTED; if (xfs_error_level >= XFS_ERRLEVEL_LOW) { xfs_alert(mp, "%s: bno %u inode %llu", - __func__, bno, dp->i_ino); + __func__, bno, I_INO(dp)); for (i = 0; i < nirecs; i++) { xfs_alert(mp, diff --git a/fs/xfs/libxfs/xfs_dir2.c b/fs/xfs/libxfs/xfs_dir2.c index 495620cc001f..0c0402a29b6b 100644 --- a/fs/xfs/libxfs/xfs_dir2.c +++ b/fs/xfs/libxfs/xfs_dir2.c @@ -244,7 +244,7 @@ xfs_dir_init( int error; ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); - error = xfs_dir_ino_validate(tp->t_mountp, pdp->i_ino); + error = xfs_dir_ino_validate(tp->t_mountp, I_INO(pdp)); if (error) return error; @@ -255,8 +255,8 @@ xfs_dir_init( args->geo = dp->i_mount->m_dir_geo; args->dp = dp; args->trans = tp; - args->owner = dp->i_ino; - error = xfs_dir2_sf_create(args, pdp->i_ino); + args->owner = I_INO(dp); + error = xfs_dir2_sf_create(args, I_INO(pdp)); kfree(args); return error; } @@ -356,7 +356,7 @@ xfs_dir_createname( args->whichfork = XFS_DATA_FORK; args->trans = tp; args->op_flags = XFS_DA_OP_ADDNAME | XFS_DA_OP_OKNOENT; - args->owner = dp->i_ino; + args->owner = I_INO(dp); rval = xfs_dir_createname_args(args); kfree(args); @@ -447,7 +447,7 @@ xfs_dir_lookup( args->whichfork = XFS_DATA_FORK; args->trans = tp; args->op_flags = XFS_DA_OP_OKNOENT; - args->owner = dp->i_ino; + args->owner = I_INO(dp); if (ci_name) args->op_flags |= XFS_DA_OP_CILOOKUP; @@ -516,7 +516,7 @@ xfs_dir_removename( args->total = total; args->whichfork = XFS_DATA_FORK; args->trans = tp; - args->owner = dp->i_ino; + args->owner = I_INO(dp); rval = xfs_dir_removename_args(args); kfree(args); return rval; @@ -576,7 +576,7 @@ xfs_dir_replace( args->total = total; args->whichfork = XFS_DATA_FORK; args->trans = tp; - args->owner = dp->i_ino; + args->owner = I_INO(dp); rval = xfs_dir_replace_args(args); kfree(args); return rval; @@ -855,7 +855,7 @@ xfs_dir_create_child( xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); xfs_assert_ilocked(dp, XFS_ILOCK_EXCL); - error = xfs_dir_createname(tp, dp, name, ip->i_ino, resblks); + error = xfs_dir_createname(tp, dp, name, I_INO(ip), resblks); if (error) { ASSERT(error != -ENOSPC); return error; @@ -919,14 +919,14 @@ xfs_dir_add_child( if (VFS_I(ip)->i_nlink == 0) { struct xfs_perag *pag; - pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino)); + pag = xfs_perag_get(mp, XFS_INODE_TO_AGNO(ip)); error = xfs_iunlink_remove(tp, pag, ip); xfs_perag_put(pag); if (error) return error; } - error = xfs_dir_createname(tp, dp, name, ip->i_ino, resblks); + error = xfs_dir_createname(tp, dp, name, I_INO(ip), resblks); if (error) return error; @@ -995,7 +995,7 @@ xfs_dir_remove_child( * get freed before the child directory is closed. If the fs * gets shrunk, this can lead to dirent inode validation errors. */ - if (dp->i_ino != tp->t_mountp->m_sb.sb_rootino) { + if (I_INO(dp) != tp->t_mountp->m_sb.sb_rootino) { error = xfs_dir_replace(tp, ip, &xfs_name_dotdot, tp->t_mountp->m_sb.sb_rootino, 0); if (error) @@ -1016,7 +1016,7 @@ xfs_dir_remove_child( if (error) return error; - error = xfs_dir_removename(tp, dp, name, ip->i_ino, resblks); + error = xfs_dir_removename(tp, dp, name, I_INO(ip), resblks); if (error) { ASSERT(error != -ENOENT); return error; @@ -1059,12 +1059,12 @@ xfs_dir_exchange_children( int error; /* Swap inode number for dirent in first parent */ - error = xfs_dir_replace(tp, dp1, name1, ip2->i_ino, spaceres); + error = xfs_dir_replace(tp, dp1, name1, I_INO(ip2), spaceres); if (error) return error; /* Swap inode number for dirent in second parent */ - error = xfs_dir_replace(tp, dp2, name2, ip1->i_ino, spaceres); + error = xfs_dir_replace(tp, dp2, name2, I_INO(ip1), spaceres); if (error) return error; @@ -1078,7 +1078,7 @@ xfs_dir_exchange_children( if (S_ISDIR(VFS_I(ip2)->i_mode)) { error = xfs_dir_replace(tp, ip2, &xfs_name_dotdot, - dp1->i_ino, spaceres); + I_INO(dp1), spaceres); if (error) return error; @@ -1102,7 +1102,7 @@ xfs_dir_exchange_children( if (S_ISDIR(VFS_I(ip1)->i_mode)) { error = xfs_dir_replace(tp, ip1, &xfs_name_dotdot, - dp2->i_ino, spaceres); + I_INO(dp2), spaceres); if (error) return error; @@ -1246,7 +1246,7 @@ xfs_dir_rename_children( ASSERT(VFS_I(du_wip->ip)->i_nlink == 0); - pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, du_wip->ip->i_ino)); + pag = xfs_perag_get(mp, XFS_INODE_TO_AGNO(du_wip->ip)); error = xfs_iunlink_remove(tp, pag, du_wip->ip); xfs_perag_put(pag); if (error) @@ -1265,7 +1265,7 @@ xfs_dir_rename_children( * to account for the ".." reference from the new entry. */ error = xfs_dir_createname(tp, target_dp, target_name, - src_ip->i_ino, spaceres); + I_INO(src_ip), spaceres); if (error) return error; @@ -1286,7 +1286,7 @@ xfs_dir_rename_children( * name at the destination directory, remove it first. */ error = xfs_dir_replace(tp, target_dp, target_name, - src_ip->i_ino, spaceres); + I_INO(src_ip), spaceres); if (error) return error; @@ -1320,7 +1320,7 @@ xfs_dir_rename_children( * directory. */ error = xfs_dir_replace(tp, src_ip, &xfs_name_dotdot, - target_dp->i_ino, spaceres); + I_INO(target_dp), spaceres); ASSERT(error != -EEXIST); if (error) return error; @@ -1358,10 +1358,10 @@ xfs_dir_rename_children( * altogether. */ if (du_wip->ip) - error = xfs_dir_replace(tp, src_dp, src_name, du_wip->ip->i_ino, + error = xfs_dir_replace(tp, src_dp, src_name, I_INO(du_wip->ip), spaceres); else - error = xfs_dir_removename(tp, src_dp, src_name, src_ip->i_ino, + error = xfs_dir_removename(tp, src_dp, src_name, I_INO(src_ip), spaceres); if (error) return error; diff --git a/fs/xfs/libxfs/xfs_dir2_node.c b/fs/xfs/libxfs/xfs_dir2_node.c index ed0b5287a44f..19cb536e53ef 100644 --- a/fs/xfs/libxfs/xfs_dir2_node.c +++ b/fs/xfs/libxfs/xfs_dir2_node.c @@ -1739,7 +1739,7 @@ xfs_dir2_node_add_datablk( fbno)) { xfs_alert(mp, "%s: dir ino %llu needed freesp block %lld for data block %lld, got %lld", - __func__, (unsigned long long)dp->i_ino, + __func__, (unsigned long long)I_INO(dp), (long long)xfs_dir2_db_to_fdb(args->geo, *dbno), (long long)*dbno, (long long)fbno); if (fblk) { diff --git a/fs/xfs/libxfs/xfs_dir2_sf.c b/fs/xfs/libxfs/xfs_dir2_sf.c index 1a67cdd6a707..0567cf8b9c1b 100644 --- a/fs/xfs/libxfs/xfs_dir2_sf.c +++ b/fs/xfs/libxfs/xfs_dir2_sf.c @@ -301,7 +301,7 @@ xfs_dir2_block_to_sf( * Skip . */ if (dep->namelen == 1 && dep->name[0] == '.') - ASSERT(be64_to_cpu(dep->inumber) == dp->i_ino); + ASSERT(be64_to_cpu(dep->inumber) == I_INO(dp)); /* * Skip .., but make sure the inode number is right. */ @@ -863,7 +863,7 @@ xfs_dir2_sf_lookup( * Special case for . */ if (args->namelen == 1 && args->name[0] == '.') { - args->inumber = dp->i_ino; + args->inumber = I_INO(dp); args->cmpresult = XFS_CMP_EXACT; args->filetype = XFS_DIR3_FT_DIR; return -EEXIST; diff --git a/fs/xfs/libxfs/xfs_exchmaps.c b/fs/xfs/libxfs/xfs_exchmaps.c index 5d28f4eac527..dcd0bd0b13b4 100644 --- a/fs/xfs/libxfs/xfs_exchmaps.c +++ b/fs/xfs/libxfs/xfs_exchmaps.c @@ -429,7 +429,7 @@ xfs_exchmaps_attr_to_sf( .geo = tp->t_mountp->m_attr_geo, .whichfork = XFS_ATTR_FORK, .trans = tp, - .owner = xmi->xmi_ip2->i_ino, + .owner = I_INO(xmi->xmi_ip2), }; struct xfs_buf *bp; int forkoff; @@ -438,7 +438,7 @@ xfs_exchmaps_attr_to_sf( if (!xfs_attr_is_leaf(xmi->xmi_ip2)) return 0; - error = xfs_attr3_leaf_read(tp, xmi->xmi_ip2, xmi->xmi_ip2->i_ino, 0, + error = xfs_attr3_leaf_read(tp, xmi->xmi_ip2, I_INO(xmi->xmi_ip2), 0, &bp); if (error) return error; @@ -461,7 +461,7 @@ xfs_exchmaps_dir_to_sf( .geo = tp->t_mountp->m_dir_geo, .whichfork = XFS_DATA_FORK, .trans = tp, - .owner = xmi->xmi_ip2->i_ino, + .owner = I_INO(xmi->xmi_ip2), }; struct xfs_dir2_sf_hdr sfh; struct xfs_buf *bp; @@ -471,7 +471,7 @@ xfs_exchmaps_dir_to_sf( if (xfs_dir2_format(&args, &error) != XFS_DIR2_FMT_BLOCK) return error; - error = xfs_dir3_block_read(tp, xmi->xmi_ip2, xmi->xmi_ip2->i_ino, &bp); + error = xfs_dir3_block_read(tp, xmi->xmi_ip2, I_INO(xmi->xmi_ip2), &bp); if (error) return error; @@ -711,7 +711,7 @@ xfs_exchmaps_estimate_overhead( return -ENOSPC; /* Can't actually reserve more than UINT_MAX blocks. */ - if (req->resblks > UINT_MAX) + if (resblks > UINT_MAX) return -ENOSPC; req->resblks = resblks; diff --git a/fs/xfs/libxfs/xfs_format.h b/fs/xfs/libxfs/xfs_format.h index 779dac59b1f3..dd0ed046fbe9 100644 --- a/fs/xfs/libxfs/xfs_format.h +++ b/fs/xfs/libxfs/xfs_format.h @@ -1276,8 +1276,12 @@ static inline bool xfs_dinode_is_metadir(const struct xfs_dinode *dip) XFS_INO_AGNO_BITS(mp) + XFS_INO_AGINO_BITS(mp) #define XFS_INO_TO_AGNO(mp,i) \ ((xfs_agnumber_t)((i) >> XFS_INO_AGINO_BITS(mp))) +#define XFS_INODE_TO_AGNO(ip) \ + XFS_INO_TO_AGNO((ip)->i_mount, I_INO(ip)) #define XFS_INO_TO_AGINO(mp,i) \ ((xfs_agino_t)(i) & XFS_INO_MASK(XFS_INO_AGINO_BITS(mp))) +#define XFS_INODE_TO_AGINO(ip) \ + XFS_INO_TO_AGINO((ip)->i_mount, I_INO(ip)) #define XFS_INO_TO_AGBNO(mp,i) \ (((xfs_agblock_t)(i) >> XFS_INO_OFFSET_BITS(mp)) & \ XFS_INO_MASK(XFS_INO_AGBNO_BITS(mp))) @@ -1285,6 +1289,8 @@ static inline bool xfs_dinode_is_metadir(const struct xfs_dinode *dip) ((int)(i) & XFS_INO_MASK(XFS_INO_OFFSET_BITS(mp))) #define XFS_INO_TO_FSB(mp,i) \ XFS_AGB_TO_FSB(mp, XFS_INO_TO_AGNO(mp,i), XFS_INO_TO_AGBNO(mp,i)) +#define XFS_INODE_TO_FSB(ip) \ + XFS_INO_TO_FSB((ip)->i_mount, I_INO(ip)) #define XFS_AGINO_TO_INO(mp,a,i) \ (((xfs_ino_t)(a) << XFS_INO_AGINO_BITS(mp)) | (i)) #define XFS_AGINO_TO_AGBNO(mp,i) ((i) >> XFS_INO_OFFSET_BITS(mp)) diff --git a/fs/xfs/libxfs/xfs_ialloc.c b/fs/xfs/libxfs/xfs_ialloc.c index dcef06ec0a02..ffcdd1f691fd 100644 --- a/fs/xfs/libxfs/xfs_ialloc.c +++ b/fs/xfs/libxfs/xfs_ialloc.c @@ -1075,7 +1075,7 @@ xfs_dialloc_check_ino( if (error) return -EAGAIN; - error = xfs_imap_to_bp(pag_mount(pag), tp, &imap, &bp); + error = xfs_read_icluster(pag, tp, imap.im_agbno, &bp); if (error) return -EAGAIN; @@ -1870,7 +1870,7 @@ xfs_dialloc_pick_ag( if (S_ISDIR(mode)) return (atomic_inc_return(&mp->m_agirotor) - 1) % mp->m_maxagi; - start_agno = XFS_INO_TO_AGNO(mp, dp->i_ino); + start_agno = XFS_INODE_TO_AGNO(dp); if (start_agno >= mp->m_maxagi) start_agno = 0; @@ -1895,7 +1895,7 @@ xfs_dialloc( struct xfs_perag *pag; struct xfs_ino_geometry *igeo = M_IGEO(mp); xfs_ino_t ino = NULLFSINO; - xfs_ino_t parent = args->pip ? args->pip->i_ino : 0; + xfs_ino_t parent = args->pip ? I_INO(args->pip) : 0; xfs_agnumber_t agno; xfs_agnumber_t start_agno; umode_t mode = args->mode & S_IFMT; @@ -2511,43 +2511,37 @@ xfs_imap( * inodes in stale state on disk. Hence we have to do a btree lookup * in all cases where an untrusted inode number is passed. */ - if (flags & XFS_IGET_UNTRUSTED) { - error = xfs_imap_lookup(pag, tp, agino, agbno, - &chunk_agbno, &offset_agbno, flags); - if (error) - return error; - goto out_map; + if (!(flags & XFS_IGET_UNTRUSTED)) { + /* + * If the inode cluster size is the same or smaller than the + * blocksize, get to the buffer by simple arithmetics. + */ + if (M_IGEO(mp)->blocks_per_cluster == 1) { + cluster_agbno = agbno; + offset = XFS_INO_TO_OFFSET(mp, ino); + ASSERT(offset < mp->m_sb.sb_inopblock); + goto out; + } + + /* + * If the inode chunks are aligned, use simple maths to find the + * location. + */ + if (M_IGEO(mp)->inoalign_mask) { + offset_agbno = agbno & M_IGEO(mp)->inoalign_mask; + chunk_agbno = agbno - offset_agbno; + goto out_map; + } + + /* + * Otherwise we have to do a btree lookup to find the location. + */ } - /* - * If the inode cluster size is the same as the blocksize or - * smaller we get to the buffer by simple arithmetics. - */ - if (M_IGEO(mp)->blocks_per_cluster == 1) { - offset = XFS_INO_TO_OFFSET(mp, ino); - ASSERT(offset < mp->m_sb.sb_inopblock); - - imap->im_blkno = xfs_agbno_to_daddr(pag, agbno); - imap->im_len = XFS_FSB_TO_BB(mp, 1); - imap->im_boffset = (unsigned short)(offset << - mp->m_sb.sb_inodelog); - return 0; - } - - /* - * If the inode chunks are aligned then use simple maths to - * find the location. Otherwise we have to do a btree - * lookup to find the location. - */ - if (M_IGEO(mp)->inoalign_mask) { - offset_agbno = agbno & M_IGEO(mp)->inoalign_mask; - chunk_agbno = agbno - offset_agbno; - } else { - error = xfs_imap_lookup(pag, tp, agino, agbno, - &chunk_agbno, &offset_agbno, flags); - if (error) - return error; - } + error = xfs_imap_lookup(pag, tp, agino, agbno, &chunk_agbno, + &offset_agbno, flags); + if (error) + return error; out_map: ASSERT(agbno >= chunk_agbno); @@ -2556,24 +2550,14 @@ out_map: M_IGEO(mp)->blocks_per_cluster); offset = ((agbno - cluster_agbno) * mp->m_sb.sb_inopblock) + XFS_INO_TO_OFFSET(mp, ino); - - imap->im_blkno = xfs_agbno_to_daddr(pag, cluster_agbno); - imap->im_len = XFS_FSB_TO_BB(mp, M_IGEO(mp)->blocks_per_cluster); +out: + imap->im_agbno = cluster_agbno; imap->im_boffset = (unsigned short)(offset << mp->m_sb.sb_inodelog); - - /* - * If the inode number maps to a block outside the bounds - * of the file system then return NULL rather than calling - * read_buf and panicing when we get an error from the - * driver. - */ - if ((imap->im_blkno + imap->im_len) > - XFS_FSB_TO_BB(mp, mp->m_sb.sb_dblocks)) { - xfs_alert(mp, - "%s: (im_blkno (0x%llx) + im_len (0x%llx)) > sb_dblocks (0x%llx)", - __func__, (unsigned long long) imap->im_blkno, - (unsigned long long) imap->im_len, - XFS_FSB_TO_BB(mp, mp->m_sb.sb_dblocks)); + if (imap->im_agbno + M_IGEO(mp)->blocks_per_cluster > + pag_group(pag)->xg_block_count) { + xfs_alert(mp, "inode cluster out of range: %u/%u > %u", + imap->im_agbno, M_IGEO(mp)->blocks_per_cluster, + pag_group(pag)->xg_block_count); return -EINVAL; } return 0; diff --git a/fs/xfs/libxfs/xfs_inode_buf.c b/fs/xfs/libxfs/xfs_inode_buf.c index 3794e5412eba..336ef843f2fe 100644 --- a/fs/xfs/libxfs/xfs_inode_buf.c +++ b/fs/xfs/libxfs/xfs_inode_buf.c @@ -123,24 +123,24 @@ const struct xfs_buf_ops xfs_inode_buf_ra_ops = { /* - * This routine is called to map an inode to the buffer containing the on-disk - * version of the inode. It returns a pointer to the buffer containing the - * on-disk inode in the bpp parameter. + * Read the inode cluster at @bno and return it in @bpp. */ int -xfs_imap_to_bp( - struct xfs_mount *mp, +xfs_read_icluster( + struct xfs_perag *pag, struct xfs_trans *tp, - struct xfs_imap *imap, + xfs_agblock_t agbno, struct xfs_buf **bpp) { + struct xfs_mount *mp = pag_mount(pag); int error; - error = xfs_trans_read_buf(mp, tp, mp->m_ddev_targp, imap->im_blkno, - imap->im_len, 0, bpp, &xfs_inode_buf_ops); + error = xfs_trans_read_buf(mp, tp, mp->m_ddev_targp, + xfs_agbno_to_daddr(pag, agbno), + XFS_FSB_TO_BB(mp, M_IGEO(mp)->blocks_per_cluster), + 0, bpp, &xfs_inode_buf_ops); if (xfs_metadata_is_sick(error)) - xfs_agno_mark_sick(mp, xfs_daddr_to_agno(mp, imap->im_blkno), - XFS_SICK_AG_INODES); + xfs_agno_mark_sick(mp, pag_agno(pag), XFS_SICK_AG_INODES); return error; } @@ -185,7 +185,7 @@ xfs_inode_from_disk( ASSERT(ip->i_cowfp == NULL); - fa = xfs_dinode_verify(ip->i_mount, ip->i_ino, from); + fa = xfs_dinode_verify(ip->i_mount, I_INO(ip), from); if (fa) { xfs_inode_verifier_error(ip, -EFSCORRUPTED, "dinode", from, sizeof(*from), fa); @@ -358,7 +358,7 @@ xfs_inode_to_disk( to->di_flags2 = cpu_to_be64(ip->i_diflags2); /* also covers the di_used_blocks union arm: */ to->di_cowextsize = cpu_to_be32(ip->i_cowextsize); - to->di_ino = cpu_to_be64(ip->i_ino); + to->di_ino = cpu_to_be64(I_INO(ip)); to->di_lsn = cpu_to_be64(lsn); memset(to->di_pad2, 0, sizeof(to->di_pad2)); uuid_copy(&to->di_uuid, &ip->i_mount->m_sb.sb_meta_uuid); diff --git a/fs/xfs/libxfs/xfs_inode_buf.h b/fs/xfs/libxfs/xfs_inode_buf.h index 8d43d2641c73..f3624532b023 100644 --- a/fs/xfs/libxfs/xfs_inode_buf.h +++ b/fs/xfs/libxfs/xfs_inode_buf.h @@ -11,16 +11,15 @@ struct xfs_dinode; /* * Inode location information. Stored in the inode and passed to - * xfs_imap_to_bp() to get a buffer and dinode for a given inode. + * xfs_read_icluster() to get a buffer and dinode for a given inode. */ struct xfs_imap { - xfs_daddr_t im_blkno; /* starting BB of inode chunk */ - unsigned short im_len; /* length in BBs of inode chunk */ - unsigned short im_boffset; /* inode offset in block in bytes */ -}; + xfs_agblock_t im_agbno; /* starting agbno of inode cluster */ + unsigned short im_boffset; /* offset in inode cluster in bytes */ +} __packed; -int xfs_imap_to_bp(struct xfs_mount *mp, struct xfs_trans *tp, - struct xfs_imap *imap, struct xfs_buf **bpp); +int xfs_read_icluster(struct xfs_perag *pag, struct xfs_trans *tp, + xfs_agblock_t agbno, struct xfs_buf **bpp); void xfs_dinode_calc_crc(struct xfs_mount *mp, struct xfs_dinode *dip); void xfs_inode_to_disk(struct xfs_inode *ip, struct xfs_dinode *to, xfs_lsn_t lsn); diff --git a/fs/xfs/libxfs/xfs_inode_fork.c b/fs/xfs/libxfs/xfs_inode_fork.c index d14a7f2f4c03..606a36526ce2 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.c +++ b/fs/xfs/libxfs/xfs_inode_fork.c @@ -87,7 +87,7 @@ xfs_iformat_local( if (unlikely(size > XFS_DFORK_SIZE(dip, ip->i_mount, whichfork))) { xfs_warn(ip->i_mount, "corrupt inode %llu (bad size %d for local fork, size = %zd).", - (unsigned long long) ip->i_ino, size, + (unsigned long long)I_INO(ip), size, XFS_DFORK_SIZE(dip, ip->i_mount, whichfork)); xfs_inode_verifier_error(ip, -EFSCORRUPTED, "xfs_iformat_local", dip, sizeof(*dip), @@ -126,7 +126,7 @@ xfs_iformat_extents( */ if (unlikely(size < 0 || size > XFS_DFORK_SIZE(dip, mp, whichfork))) { xfs_warn(ip->i_mount, "corrupt inode %llu ((a)extents = %llu).", - ip->i_ino, nex); + I_INO(ip), nex); xfs_inode_verifier_error(ip, -EFSCORRUPTED, "xfs_iformat_extents(1)", dip, sizeof(*dip), __this_address); @@ -205,7 +205,7 @@ xfs_iformat_btree( ifp->if_nextents > ip->i_nblocks) || level == 0 || level > XFS_BM_MAXLEVELS(mp, whichfork)) { xfs_warn(mp, "corrupt inode %llu (btree).", - (unsigned long long) ip->i_ino); + (unsigned long long)I_INO(ip)); xfs_inode_verifier_error(ip, -EFSCORRUPTED, "xfs_iformat_btree", dfp, size, __this_address); diff --git a/fs/xfs/libxfs/xfs_inode_util.c b/fs/xfs/libxfs/xfs_inode_util.c index 82be54b6f8d3..258ac3d0d486 100644 --- a/fs/xfs/libxfs/xfs_inode_util.c +++ b/fs/xfs/libxfs/xfs_inode_util.c @@ -464,10 +464,9 @@ xfs_iunlink_insert_inode( struct xfs_buf *agibp, struct xfs_inode *ip) { - struct xfs_mount *mp = tp->t_mountp; struct xfs_agi *agi = agibp->b_addr; xfs_agino_t next_agino; - xfs_agino_t agino = XFS_INO_TO_AGINO(mp, ip->i_ino); + xfs_agino_t agino = XFS_INODE_TO_AGINO(ip); short bucket_index = agino % XFS_AGI_UNLINKED_BUCKETS; int error; @@ -531,7 +530,7 @@ xfs_iunlink( ASSERT(VFS_I(ip)->i_mode != 0); trace_xfs_iunlink(ip); - pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino)); + pag = xfs_perag_get(mp, XFS_INODE_TO_AGNO(ip)); /* Get the agi buffer first. It ensures lock ordering on the list. */ error = xfs_read_agi(pag, tp, 0, &agibp); @@ -553,7 +552,7 @@ xfs_iunlink_remove_inode( { struct xfs_mount *mp = tp->t_mountp; struct xfs_agi *agi = agibp->b_addr; - xfs_agino_t agino = XFS_INO_TO_AGINO(mp, ip->i_ino); + xfs_agino_t agino = XFS_INODE_TO_AGINO(ip); xfs_agino_t head_agino; short bucket_index = agino % XFS_AGI_UNLINKED_BUCKETS; int error; @@ -655,7 +654,7 @@ xfs_droplink( if (inode->i_nlink == 0) { xfs_info_ratelimited(tp->t_mountp, "Inode 0x%llx link count dropped below zero. Pinning link count.", - ip->i_ino); + I_INO(ip)); set_nlink(inode, XFS_NLINK_PINNED); } if (inode->i_nlink != XFS_NLINK_PINNED) @@ -684,7 +683,7 @@ xfs_bumplink( if (inode->i_nlink == XFS_NLINK_PINNED - 1) xfs_info_ratelimited(tp->t_mountp, "Inode 0x%llx link count exceeded maximum. Pinning link count.", - ip->i_ino); + I_INO(ip)); if (inode->i_nlink != XFS_NLINK_PINNED) inc_nlink(inode); @@ -708,7 +707,7 @@ xfs_inode_uninit( * makes the AGI lock -> unlinked list modification order the same as * used in O_TMPFILE creation. */ - error = xfs_difree(tp, pag, ip->i_ino, xic); + error = xfs_difree(tp, pag, I_INO(ip), xic); if (error) return error; diff --git a/fs/xfs/libxfs/xfs_log_format.h b/fs/xfs/libxfs/xfs_log_format.h index 3f5a24dda907..a4e1b3eb425c 100644 --- a/fs/xfs/libxfs/xfs_log_format.h +++ b/fs/xfs/libxfs/xfs_log_format.h @@ -52,6 +52,19 @@ typedef uint32_t xlog_tid_t; #define CYCLE_LSN(lsn) ((uint)((lsn)>>32)) #define BLOCK_LSN(lsn) ((uint)(lsn)) +/* + * By comparing each component, we don't have to worry about extra endian issues + * in treating two 32 bit numbers as one 64 bit number + */ +static inline xfs_lsn_t XFS_LSN_CMP(xfs_lsn_t lsn1, xfs_lsn_t lsn2) +{ + if (CYCLE_LSN(lsn1) != CYCLE_LSN(lsn2)) + return CYCLE_LSN(lsn1) < CYCLE_LSN(lsn2) ? -999 : 999; + if (BLOCK_LSN(lsn1) != BLOCK_LSN(lsn2)) + return BLOCK_LSN(lsn1) < BLOCK_LSN(lsn2) ? -999 : 999; + return 0; +} + /* this is used in a spot where we might otherwise double-endian-flip */ #define CYCLE_LSN_DISK(lsn) (((__be32 *)&(lsn))[0]) diff --git a/fs/xfs/libxfs/xfs_metadir.c b/fs/xfs/libxfs/xfs_metadir.c index 3e5c61188927..74c4596ee4cf 100644 --- a/fs/xfs/libxfs/xfs_metadir.c +++ b/fs/xfs/libxfs/xfs_metadir.c @@ -95,7 +95,7 @@ xfs_metadir_lookup( .hashval = xfs_dir2_hashname(mp, xname), .whichfork = XFS_DATA_FORK, .op_flags = XFS_DA_OP_OKNOENT, - .owner = dp->i_ino, + .owner = I_INO(dp), }; int error; diff --git a/fs/xfs/libxfs/xfs_parent.c b/fs/xfs/libxfs/xfs_parent.c index 3509cc4b2175..8d111c9b6527 100644 --- a/fs/xfs/libxfs/xfs_parent.c +++ b/fs/xfs/libxfs/xfs_parent.c @@ -23,7 +23,6 @@ #include "xfs_attr_sf.h" #include "xfs_bmap.h" #include "xfs_defer.h" -#include "xfs_log.h" #include "xfs_xattr.h" #include "xfs_parent.h" #include "xfs_trans_space.h" @@ -202,7 +201,7 @@ xfs_parent_addname( xfs_inode_to_parent_rec(&ppargs->rec, dp); xfs_parent_da_args_init(&ppargs->args, tp, &ppargs->rec, child, - child->i_ino, parent_name); + I_INO(child), parent_name); return xfs_attr_setname(&ppargs->args, 0); } @@ -224,7 +223,7 @@ xfs_parent_removename( xfs_inode_to_parent_rec(&ppargs->rec, dp); xfs_parent_da_args_init(&ppargs->args, tp, &ppargs->rec, child, - child->i_ino, parent_name); + I_INO(child), parent_name); return xfs_attr_removename(&ppargs->args); } @@ -248,7 +247,7 @@ xfs_parent_replacename( xfs_inode_to_parent_rec(&ppargs->rec, old_dp); xfs_parent_da_args_init(&ppargs->args, tp, &ppargs->rec, child, - child->i_ino, old_name); + I_INO(child), old_name); xfs_inode_to_parent_rec(&ppargs->new_rec, new_dp); @@ -312,7 +311,7 @@ xfs_parent_lookup( struct xfs_da_args *scratch) { memset(scratch, 0, sizeof(struct xfs_da_args)); - xfs_parent_da_args_init(scratch, tp, pptr, ip, ip->i_ino, parent_name); + xfs_parent_da_args_init(scratch, tp, pptr, ip, I_INO(ip), parent_name); return xfs_attr_get_ilocked(scratch); } diff --git a/fs/xfs/libxfs/xfs_parent.h b/fs/xfs/libxfs/xfs_parent.h index b8036527cdc7..8eb4de9c5f1a 100644 --- a/fs/xfs/libxfs/xfs_parent.h +++ b/fs/xfs/libxfs/xfs_parent.h @@ -34,7 +34,7 @@ xfs_inode_to_parent_rec( struct xfs_parent_rec *rec, const struct xfs_inode *dp) { - xfs_parent_rec_init(rec, dp->i_ino, VFS_IC(dp)->i_generation); + xfs_parent_rec_init(rec, I_INO(dp), VFS_IC(dp)->i_generation); } extern struct kmem_cache *xfs_parent_args_cache; diff --git a/fs/xfs/libxfs/xfs_rmap.c b/fs/xfs/libxfs/xfs_rmap.c index e78133c908ca..34d218de21a9 100644 --- a/fs/xfs/libxfs/xfs_rmap.c +++ b/fs/xfs/libxfs/xfs_rmap.c @@ -2780,7 +2780,7 @@ xfs_rmap_map_extent( if (whichfork != XFS_ATTR_FORK && xfs_is_reflink_inode(ip)) type = XFS_RMAP_MAP_SHARED; - __xfs_rmap_add(tp, type, ip->i_ino, isrt, whichfork, PREV); + __xfs_rmap_add(tp, type, I_INO(ip), isrt, whichfork, PREV); } /* Unmap an extent out of a file. */ @@ -2800,7 +2800,7 @@ xfs_rmap_unmap_extent( if (whichfork != XFS_ATTR_FORK && xfs_is_reflink_inode(ip)) type = XFS_RMAP_UNMAP_SHARED; - __xfs_rmap_add(tp, type, ip->i_ino, isrt, whichfork, PREV); + __xfs_rmap_add(tp, type, I_INO(ip), isrt, whichfork, PREV); } /* @@ -2826,7 +2826,7 @@ xfs_rmap_convert_extent( if (whichfork != XFS_ATTR_FORK && xfs_is_reflink_inode(ip)) type = XFS_RMAP_CONVERT_SHARED; - __xfs_rmap_add(tp, type, ip->i_ino, isrt, whichfork, PREV); + __xfs_rmap_add(tp, type, I_INO(ip), isrt, whichfork, PREV); } /* Schedule the creation of an rmap for non-file data. */ diff --git a/fs/xfs/libxfs/xfs_rmap.h b/fs/xfs/libxfs/xfs_rmap.h index 5f39f6e53cd1..4afa33575ce2 100644 --- a/fs/xfs/libxfs/xfs_rmap.h +++ b/fs/xfs/libxfs/xfs_rmap.h @@ -21,6 +21,8 @@ xfs_rmap_ino_bmbt_owner( if (whichfork == XFS_ATTR_FORK) oi->oi_flags |= XFS_OWNER_INFO_ATTR_FORK; } +#define xfs_rmap_inode_bmbt_owner(oi, ip, whichfork) \ + xfs_rmap_ino_bmbt_owner(oi, I_INO(ip), whichfork) static inline void xfs_rmap_ino_owner( @@ -35,6 +37,8 @@ xfs_rmap_ino_owner( if (whichfork == XFS_ATTR_FORK) oi->oi_flags |= XFS_OWNER_INFO_ATTR_FORK; } +#define xfs_rmap_inode_owner(oi, ip, whichfork, offset) \ + xfs_rmap_ino_owner(oi, I_INO(ip), whichfork, offset) static inline bool xfs_rmap_should_skip_owner_update( diff --git a/fs/xfs/libxfs/xfs_rtbitmap.c b/fs/xfs/libxfs/xfs_rtbitmap.c index bc4c0a99f4dd..e62786e90f7b 100644 --- a/fs/xfs/libxfs/xfs_rtbitmap.c +++ b/fs/xfs/libxfs/xfs_rtbitmap.c @@ -214,7 +214,7 @@ xfs_rtbuf_get( if (xfs_has_rtgroups(mp)) { struct xfs_rtbuf_blkinfo *hdr = bp->b_addr; - if (hdr->rt_owner != cpu_to_be64(ip->i_ino)) { + if (hdr->rt_owner != cpu_to_be64(I_INO(ip))) { xfs_buf_mark_corrupt(bp); xfs_trans_brelse(args->tp, bp); xfs_rtginode_mark_sick(args->rtg, type); @@ -1409,7 +1409,7 @@ xfs_rtfile_initialize_block( hdr->rt_magic = cpu_to_be32(XFS_RTBITMAP_MAGIC); else hdr->rt_magic = cpu_to_be32(XFS_RTSUMMARY_MAGIC); - hdr->rt_owner = cpu_to_be64(ip->i_ino); + hdr->rt_owner = cpu_to_be64(I_INO(ip)); hdr->rt_blkno = cpu_to_be64(XFS_FSB_TO_DADDR(mp, fsbno)); hdr->rt_lsn = 0; uuid_copy(&hdr->rt_uuid, &mp->m_sb.sb_meta_uuid); diff --git a/fs/xfs/libxfs/xfs_rtrefcount_btree.c b/fs/xfs/libxfs/xfs_rtrefcount_btree.c index c1518267eb17..f27b80a199ba 100644 --- a/fs/xfs/libxfs/xfs_rtrefcount_btree.c +++ b/fs/xfs/libxfs/xfs_rtrefcount_btree.c @@ -602,7 +602,7 @@ xfs_rtrefcountbt_from_disk( rblocklen = xfs_rtrefcount_broot_space(mp, dblock); xfs_btree_init_block(mp, rblock, &xfs_rtrefcountbt_ops, 0, 0, - ip->i_ino); + I_INO(ip)); rblock->bb_level = dblock->bb_level; rblock->bb_numrecs = dblock->bb_numrecs; @@ -751,7 +751,7 @@ xfs_rtrefcountbt_create( xfs_rtrefcount_broot_space_calc(mp, 0, 0)); if (broot) xfs_btree_init_block(mp, broot, &xfs_rtrefcountbt_ops, 0, 0, - ip->i_ino); + I_INO(ip)); xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE | XFS_ILOG_DBROOT); return 0; } diff --git a/fs/xfs/libxfs/xfs_rtrmap_btree.c b/fs/xfs/libxfs/xfs_rtrmap_btree.c index 00557b7ef298..c264bc5651c0 100644 --- a/fs/xfs/libxfs/xfs_rtrmap_btree.c +++ b/fs/xfs/libxfs/xfs_rtrmap_btree.c @@ -839,7 +839,7 @@ xfs_rtrmapbt_from_disk( unsigned int numrecs; unsigned int maxrecs; - xfs_btree_init_block(mp, rblock, &xfs_rtrmapbt_ops, 0, 0, ip->i_ino); + xfs_btree_init_block(mp, rblock, &xfs_rtrmapbt_ops, 0, 0, I_INO(ip)); rblock->bb_level = dblock->bb_level; rblock->bb_numrecs = dblock->bb_numrecs; @@ -981,7 +981,7 @@ xfs_rtrmapbt_create( broot = xfs_broot_realloc(ifp, xfs_rtrmap_broot_space_calc(mp, 0, 0)); if (broot) xfs_btree_init_block(mp, broot, &xfs_rtrmapbt_ops, 0, 0, - ip->i_ino); + I_INO(ip)); xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE | XFS_ILOG_DBROOT); return 0; diff --git a/fs/xfs/libxfs/xfs_symlink_remote.c b/fs/xfs/libxfs/xfs_symlink_remote.c index f9a5966d8048..b0dc3888bf1b 100644 --- a/fs/xfs/libxfs/xfs_symlink_remote.c +++ b/fs/xfs/libxfs/xfs_symlink_remote.c @@ -196,7 +196,7 @@ xfs_symlink_local_to_remote( bp->b_ops = &xfs_symlink_buf_ops; buf = bp->b_addr; - buf += xfs_symlink_hdr_set(mp, ip->i_ino, 0, ifp->if_bytes, bp); + buf += xfs_symlink_hdr_set(mp, I_INO(ip), 0, ifp->if_bytes, bp); memcpy(buf, ifp->if_data, ifp->if_bytes); xfs_trans_log_buf(tp, bp, 0, sizeof(struct xfs_dsymlink_hdr) + ifp->if_bytes - 1); @@ -277,13 +277,13 @@ xfs_symlink_remote_read( cur_chunk = bp->b_addr; if (xfs_has_crc(mp)) { - if (!xfs_symlink_hdr_ok(ip->i_ino, offset, + if (!xfs_symlink_hdr_ok(I_INO(ip), offset, byte_cnt, bp)) { xfs_inode_mark_sick(ip, XFS_SICK_INO_SYMLINK); error = -EFSCORRUPTED; xfs_alert(mp, "symlink header does not match required off/len/owner (0x%x/0x%x,0x%llx)", - offset, byte_cnt, ip->i_ino); + offset, byte_cnt, I_INO(ip)); xfs_buf_relse(bp); goto out; diff --git a/fs/xfs/scrub/agheader_repair.c b/fs/xfs/scrub/agheader_repair.c index ae9ed5f280d0..7114c9a44182 100644 --- a/fs/xfs/scrub/agheader_repair.c +++ b/fs/xfs/scrub/agheader_repair.c @@ -1116,9 +1116,7 @@ xrep_iunlink_igrab( struct xfs_perag *pag, struct xfs_inode *ip) { - struct xfs_mount *mp = pag_mount(pag); - - if (XFS_INO_TO_AGNO(mp, ip->i_ino) != pag_agno(pag)) + if (XFS_INODE_TO_AGNO(ip) != pag_agno(pag)) return false; if (!xfs_inode_on_unlinked_list(ip)) @@ -1136,18 +1134,14 @@ xrep_iunlink_visit( struct xrep_agi *ragi, unsigned int batch_idx) { - struct xfs_mount *mp = ragi->sc->mp; struct xfs_inode *ip = ragi->lookup_batch[batch_idx]; - xfs_agino_t agino; - unsigned int bucket; + xfs_agino_t agino = XFS_INODE_TO_AGINO(ip); + unsigned int bucket = agino % XFS_AGI_UNLINKED_BUCKETS; int error; - ASSERT(XFS_INO_TO_AGNO(mp, ip->i_ino) == pag_agno(ragi->sc->sa.pag)); + ASSERT(XFS_INODE_TO_AGNO(ip) == pag_agno(ragi->sc->sa.pag)); ASSERT(xfs_inode_on_unlinked_list(ip)); - agino = XFS_INO_TO_AGINO(mp, ip->i_ino); - bucket = agino % XFS_AGI_UNLINKED_BUCKETS; - trace_xrep_iunlink_visit(ragi->sc->sa.pag, bucket, ragi->iunlink_heads[bucket], ip); @@ -1213,10 +1207,10 @@ xrep_iunlink_mark_incore( * us to see this inode, so another lookup from the * same index will not find it again. */ - if (XFS_INO_TO_AGNO(mp, ip->i_ino) != pag_agno(pag)) + if (XFS_INODE_TO_AGNO(ip) != pag_agno(pag)) continue; - first_index = XFS_INO_TO_AGINO(mp, ip->i_ino + 1); - if (first_index < XFS_INO_TO_AGINO(mp, ip->i_ino)) + first_index = XFS_INO_TO_AGINO(mp, I_INO(ip) + 1); + if (first_index < XFS_INODE_TO_AGINO(ip)) done = true; } diff --git a/fs/xfs/scrub/attr.c b/fs/xfs/scrub/attr.c index 390ac2e11ee0..d5226939c810 100644 --- a/fs/xfs/scrub/attr.c +++ b/fs/xfs/scrub/attr.c @@ -182,7 +182,7 @@ xchk_xattr_actor( .namelen = namelen, .trans = sc->tp, .valuelen = valuelen, - .owner = ip->i_ino, + .owner = I_INO(ip), }; struct xchk_xattr_buf *ab; int error = 0; @@ -199,7 +199,7 @@ xchk_xattr_actor( if (attr_flags & XFS_ATTR_INCOMPLETE) { /* Incomplete attr key, just mark the inode for preening. */ - xchk_ino_set_preen(sc, ip->i_ino); + xchk_ino_set_preen(sc, I_INO(ip)); return 0; } diff --git a/fs/xfs/scrub/attr_repair.c b/fs/xfs/scrub/attr_repair.c index 7d00a1ce0bac..be627ab655ad 100644 --- a/fs/xfs/scrub/attr_repair.c +++ b/fs/xfs/scrub/attr_repair.c @@ -329,7 +329,7 @@ xrep_xattr_salvage_remote_attr( .dp = rx->sc->ip, .index = ent_idx, .geo = rx->sc->mp->m_attr_geo, - .owner = rx->sc->ip->i_ino, + .owner = I_INO(rx->sc->ip), .attr_filter = ent->flags & XFS_ATTR_NSP_ONDISK_MASK, .namelen = rentry->namelen, .name = rentry->name, @@ -590,7 +590,7 @@ xrep_xattr_recover_block( * as much as we can from the block. */ if (info->magic == cpu_to_be16(XFS_ATTR3_LEAF_MAGIC) && xrep_buf_verify_struct(bp, &xfs_attr3_leaf_buf_ops) && - xfs_attr3_leaf_header_check(bp, rx->sc->ip->i_ino) == NULL) + xfs_attr3_leaf_header_check(bp, I_INO(rx->sc->ip)) == NULL) error = xrep_xattr_recover_leaf(rx, bp); /* @@ -617,7 +617,7 @@ xrep_xattr_insert_rec( .attr_filter = key->flags, .namelen = key->namelen, .valuelen = key->valuelen, - .owner = rx->sc->ip->i_ino, + .owner = I_INO(rx->sc->ip), .geo = rx->sc->mp->m_attr_geo, .whichfork = XFS_ATTR_FORK, .op_flags = XFS_DA_OP_OKNOENT, @@ -982,7 +982,7 @@ xrep_xattr_fork_remove( xfs_emerg(sc->mp, "inode 0x%llx attr fork still has %llu attr extents, format %d?!", - ip->i_ino, ifp->if_nextents, ifp->if_format); + I_INO(ip), ifp->if_nextents, ifp->if_format); for_each_xfs_iext(ifp, &icur, &irec) { xfs_err(sc->mp, "[%u]: startoff %llu startblock %llu blockcount %llu state %u", @@ -1116,7 +1116,7 @@ xrep_xattr_replay_pptr_update( trace_xrep_xattr_replay_parentadd(sc->tempip, xname, &pptr->pptr_rec); - error = xfs_parent_set(sc->tempip, sc->ip->i_ino, xname, + error = xfs_parent_set(sc->tempip, I_INO(sc->ip), xname, &pptr->pptr_rec, &rx->pptr_args); ASSERT(error != -EEXIST); return error; @@ -1125,7 +1125,7 @@ xrep_xattr_replay_pptr_update( trace_xrep_xattr_replay_parentremove(sc->tempip, xname, &pptr->pptr_rec); - error = xfs_parent_unset(sc->tempip, sc->ip->i_ino, xname, + error = xfs_parent_unset(sc->tempip, I_INO(sc->ip), xname, &pptr->pptr_rec, &rx->pptr_args); ASSERT(error != -ENOATTR); return error; @@ -1256,7 +1256,7 @@ xrep_xattr_live_dirent_update( * repairing, so stash the update for replay against the temporary * file. */ - if (p->ip->i_ino != sc->ip->i_ino) + if (I_INO(p->ip) != I_INO(sc->ip)) return NOTIFY_DONE; mutex_lock(&rx->lock); @@ -1295,7 +1295,7 @@ xrep_xattr_swap_prep( .whichfork = XFS_ATTR_FORK, .trans = sc->tp, .total = 1, - .owner = sc->ip->i_ino, + .owner = I_INO(sc->ip), }; error = xfs_attr_shortform_to_leaf(&args); diff --git a/fs/xfs/scrub/bmap.c b/fs/xfs/scrub/bmap.c index d40534bf9ef9..70028da1aacc 100644 --- a/fs/xfs/scrub/bmap.c +++ b/fs/xfs/scrub/bmap.c @@ -205,7 +205,7 @@ xchk_bmap_xref_rmap( { struct xfs_rmap_irec rmap; unsigned long long rmap_end; - uint64_t owner = info->sc->ip->i_ino; + uint64_t owner = I_INO(info->sc->ip); if (xchk_skip_xref(info->sc->sm)) return; @@ -352,7 +352,7 @@ xchk_bmap_rt_iextent_xref( case XFS_DATA_FORK: xchk_bmap_xref_rmap(info, irec, rgbno); if (!xfs_is_reflink_inode(info->sc->ip)) { - xfs_rmap_ino_owner(&oinfo, info->sc->ip->i_ino, + xfs_rmap_inode_owner(&oinfo, info->sc->ip, info->whichfork, irec->br_startoff); xchk_xref_is_only_rt_owned_by(info->sc, rgbno, irec->br_blockcount, &oinfo); @@ -407,7 +407,7 @@ xchk_bmap_iextent_xref( case XFS_DATA_FORK: xchk_bmap_xref_rmap(info, irec, agbno); if (!xfs_is_reflink_inode(info->sc->ip)) { - xfs_rmap_ino_owner(&oinfo, info->sc->ip->i_ino, + xfs_rmap_inode_owner(&oinfo, info->sc->ip, info->whichfork, irec->br_startoff); xchk_xref_is_only_owned_by(info->sc, agbno, irec->br_blockcount, &oinfo); @@ -419,7 +419,7 @@ xchk_bmap_iextent_xref( break; case XFS_ATTR_FORK: xchk_bmap_xref_rmap(info, irec, agbno); - xfs_rmap_ino_owner(&oinfo, info->sc->ip->i_ino, + xfs_rmap_inode_owner(&oinfo, info->sc->ip, info->whichfork, irec->br_startoff); xchk_xref_is_only_owned_by(info->sc, agbno, irec->br_blockcount, &oinfo); @@ -543,7 +543,7 @@ xchk_bmapbt_rec( for (i = 0; i < bs->cur->bc_nlevels - 1; i++) { block = xfs_btree_get_block(bs->cur, i, &bp); owner = be64_to_cpu(block->bb_u.l.bb_owner); - if (owner != ip->i_ino) + if (owner != I_INO(ip)) xchk_fblock_set_corrupt(bs->sc, info->whichfork, 0); } @@ -600,7 +600,7 @@ xchk_bmap_btree( /* Check the btree structure. */ cur = xfs_bmbt_init_cursor(mp, sc->tp, ip, whichfork); - xfs_rmap_ino_bmbt_owner(&oinfo, ip->i_ino, whichfork); + xfs_rmap_inode_bmbt_owner(&oinfo, ip, whichfork); error = xchk_btree(sc, cur, xchk_bmapbt_rec, &oinfo, info); xfs_btree_del_cursor(cur, error); out: @@ -628,7 +628,7 @@ xchk_bmap_check_rmap( bool have_map; /* Is this even the right fork? */ - if (rec->rm_owner != sc->ip->i_ino) + if (rec->rm_owner != I_INO(sc->ip)) return 0; if ((sbcri->whichfork == XFS_ATTR_FORK) ^ !!(rec->rm_flags & XFS_RMAP_ATTR_FORK)) @@ -1003,7 +1003,7 @@ xchk_bmap_iext_iter( */ if (nr > 1 && info->whichfork != XFS_COW_FORK && howmany_64(irec->br_blockcount, XFS_MAX_BMBT_EXTLEN) < nr) - xchk_ino_set_preen(info->sc, info->sc->ip->i_ino); + xchk_ino_set_preen(info->sc, I_INO(info->sc->ip)); return true; } @@ -1040,7 +1040,7 @@ xchk_bmap( case XFS_COW_FORK: /* No CoW forks filesystem doesn't support out of place writes */ if (!xfs_has_reflink(mp) && !xfs_has_zoned(mp)) { - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); return 0; } break; @@ -1052,7 +1052,7 @@ xchk_bmap( * attr here. */ if (!xfs_has_attr(mp)) - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); break; default: ASSERT(whichfork == XFS_DATA_FORK); @@ -1137,7 +1137,7 @@ xchk_bmap_data( int error; if (xchk_file_looks_zapped(sc, XFS_SICK_INO_BMBTD_ZAPPED)) { - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); return 0; } @@ -1165,7 +1165,7 @@ xchk_bmap_attr( * returning immediately. */ if (xchk_file_looks_zapped(sc, XFS_SICK_INO_BMBTA_ZAPPED)) { - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); return 0; } diff --git a/fs/xfs/scrub/bmap_repair.c b/fs/xfs/scrub/bmap_repair.c index 822a4af43833..eabffba47776 100644 --- a/fs/xfs/scrub/bmap_repair.c +++ b/fs/xfs/scrub/bmap_repair.c @@ -252,7 +252,7 @@ xrep_bmap_walk_rmap( if (xchk_should_terminate(rb->sc, &error)) return error; - if (rec->rm_owner != rb->sc->ip->i_ino) + if (rec->rm_owner != I_INO(rb->sc->ip)) return 0; error = xrep_bmap_check_fork_rmap(rb, cur, rec); @@ -416,7 +416,7 @@ xrep_bmap_walk_rtrmap( return error; /* Skip extents which are not owned by this inode and fork. */ - if (rec->rm_owner != rb->sc->ip->i_ino) + if (rec->rm_owner != I_INO(rb->sc->ip)) return 0; error = xrep_bmap_check_rtfork_rmap(rb->sc, cur, rec); @@ -760,7 +760,7 @@ xrep_bmap_build_new_fork( * Prepare to construct the new fork by initializing the new btree * structure and creating a fake ifork in the ifakeroot structure. */ - xfs_rmap_ino_bmbt_owner(&oinfo, sc->ip->i_ino, rb->whichfork); + xfs_rmap_inode_bmbt_owner(&oinfo, sc->ip, rb->whichfork); error = xrep_newbt_init_inode(&rb->new_bmapbt, sc, rb->whichfork, &oinfo); if (error) @@ -833,7 +833,7 @@ xrep_bmap_remove_old_tree( struct xfs_owner_info oinfo; /* Free the old bmbt blocks if they're not in use. */ - xfs_rmap_ino_bmbt_owner(&oinfo, sc->ip->i_ino, rb->whichfork); + xfs_rmap_inode_bmbt_owner(&oinfo, sc->ip, rb->whichfork); return xrep_reap_fsblocks(sc, &rb->old_bmbt_blocks, &oinfo); } diff --git a/fs/xfs/scrub/common.c b/fs/xfs/scrub/common.c index 3d40cb0b2496..e5233e31abb7 100644 --- a/fs/xfs/scrub/common.c +++ b/fs/xfs/scrub/common.c @@ -320,12 +320,12 @@ xchk_ino_set_corrupt( /* Record a corruption while cross-referencing with an inode. */ void -xchk_ino_xref_set_corrupt( +xchk_ip_xref_set_corrupt( struct xfs_scrub *sc, - xfs_ino_t ino) + struct xfs_inode *ip) { sc->sm->sm_flags |= XFS_SCRUB_OFLAG_XCORRUPT; - trace_xchk_ino_error(sc, ino, __return_address); + trace_xchk_ino_error(sc, I_INO(ip), __return_address); } /* Record corruption in a block indexed by a file fork. */ @@ -1110,7 +1110,7 @@ xchk_install_live_inode( struct xfs_inode *ip) { if (!igrab(VFS_I(ip))) { - xchk_ino_set_corrupt(sc, ip->i_ino); + xchk_ip_set_corrupt(sc, ip); return -EFSCORRUPTED; } @@ -1141,7 +1141,7 @@ xchk_iget_for_scrubbing( ASSERT(sc->tp == NULL); /* We want to scan the inode we already had opened. */ - if (sc->sm->sm_ino == 0 || sc->sm->sm_ino == ip_in->i_ino) + if (sc->sm->sm_ino == 0 || sc->sm->sm_ino == I_INO(ip_in)) return xchk_install_live_inode(sc, ip_in); /* @@ -1439,13 +1439,13 @@ xchk_metadata_inode_forks( /* Metadata inodes don't live on the rt device. */ if (sc->ip->i_diflags & XFS_DIFLAG_REALTIME) { - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); return 0; } /* They should never participate in reflink. */ if (xfs_is_reflink_inode(sc->ip)) { - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); return 0; } @@ -1462,7 +1462,7 @@ xchk_metadata_inode_forks( &error)) return error; if (shared) - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); } /* @@ -1471,7 +1471,7 @@ xchk_metadata_inode_forks( */ if (xfs_inode_hasattr(sc->ip)) { if (!xfs_has_metadir(sc->mp)) { - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); return 0; } @@ -1573,7 +1573,7 @@ xchk_inode_is_allocated( * This isn't the inode we want. */ spin_lock(&ip->i_flags_lock); - if (ip->i_ino != ino) + if (I_INO(ip) != ino) goto out_skip; trace_xchk_inode_is_allocated(ip); @@ -1681,7 +1681,7 @@ bool xchk_inode_is_sb_rooted(const struct xfs_inode *ip) { return xchk_inode_is_dirtree_root(ip) || - xfs_is_sb_inum(ip->i_mount, ip->i_ino); + xfs_is_sb_inum(ip->i_mount, I_INO(ip)); } /* What is the root directory inumber for this inode? */ @@ -1691,8 +1691,8 @@ xchk_inode_rootdir_inum(const struct xfs_inode *ip) struct xfs_mount *mp = ip->i_mount; if (xfs_is_metadir_inode(ip)) - return mp->m_metadirip->i_ino; - return mp->m_rootip->i_ino; + return I_INO(mp->m_metadirip); + return I_INO(mp->m_rootip); } static int diff --git a/fs/xfs/scrub/common.h b/fs/xfs/scrub/common.h index b494d747c008..9d627fd50687 100644 --- a/fs/xfs/scrub/common.h +++ b/fs/xfs/scrub/common.h @@ -32,6 +32,8 @@ void xchk_set_corrupt(struct xfs_scrub *sc); void xchk_block_set_corrupt(struct xfs_scrub *sc, struct xfs_buf *bp); void xchk_ino_set_corrupt(struct xfs_scrub *sc, xfs_ino_t ino); +#define xchk_ip_set_corrupt(_sc, _ip) \ + xchk_ino_set_corrupt((_sc), I_INO(_ip)) void xchk_fblock_set_corrupt(struct xfs_scrub *sc, int whichfork, xfs_fileoff_t offset); #ifdef CONFIG_XFS_QUOTA @@ -41,8 +43,8 @@ void xchk_qcheck_set_corrupt(struct xfs_scrub *sc, unsigned int dqtype, void xchk_block_xref_set_corrupt(struct xfs_scrub *sc, struct xfs_buf *bp); -void xchk_ino_xref_set_corrupt(struct xfs_scrub *sc, - xfs_ino_t ino); +void xchk_ip_xref_set_corrupt(struct xfs_scrub *sc, + struct xfs_inode *ip); void xchk_fblock_xref_set_corrupt(struct xfs_scrub *sc, int whichfork, xfs_fileoff_t offset); diff --git a/fs/xfs/scrub/cow_repair.c b/fs/xfs/scrub/cow_repair.c index c25716fc4fee..0075b6d5a1b5 100644 --- a/fs/xfs/scrub/cow_repair.c +++ b/fs/xfs/scrub/cow_repair.c @@ -420,8 +420,7 @@ xrep_cow_alloc( if (error) return error; - error = xfs_alloc_vextent_start_ag(&args, - XFS_INO_TO_FSB(sc->mp, sc->ip->i_ino)); + error = xfs_alloc_vextent_start_ag(&args, XFS_INODE_TO_FSB(sc->ip)); if (error) return error; if (args.fsbno == NULLFSBLOCK) diff --git a/fs/xfs/scrub/dabtree.c b/fs/xfs/scrub/dabtree.c index c2d6ad59d03e..d4e5afbde856 100644 --- a/fs/xfs/scrub/dabtree.c +++ b/fs/xfs/scrub/dabtree.c @@ -393,7 +393,7 @@ xchk_da_btree_block( /* Check the owner. */ if (xfs_has_crc(ip->i_mount)) { owner = be64_to_cpu(hdr3->owner); - if (owner != ip->i_ino) + if (owner != I_INO(ip)) xchk_da_set_corrupt(ds, level); } @@ -524,7 +524,7 @@ xchk_da_btree( ds->dargs.whichfork = whichfork; ds->dargs.trans = sc->tp; ds->dargs.op_flags = XFS_DA_OP_OKNOENT; - ds->dargs.owner = sc->ip->i_ino; + ds->dargs.owner = I_INO(sc->ip); ds->state = xfs_da_state_alloc(&ds->dargs); ds->sc = sc; ds->private = private; diff --git a/fs/xfs/scrub/dir.c b/fs/xfs/scrub/dir.c index 09715a4aa154..2a037aae904d 100644 --- a/fs/xfs/scrub/dir.c +++ b/fs/xfs/scrub/dir.c @@ -187,11 +187,11 @@ xchk_dir_check_pptr_fast( if (!lockmode) { struct xchk_dirent save_de = { .namelen = name->len, - .ino = ip->i_ino, + .ino = I_INO(ip), }; /* Couldn't lock the inode, so save the dirent for later. */ - trace_xchk_dir_defer(sc->ip, name, ip->i_ino); + trace_xchk_dir_defer(sc->ip, name, I_INO(ip)); error = xfblob_storename(sd->dir_names, &save_de.name_cookie, name); @@ -254,14 +254,14 @@ xchk_dir_actor( if (xfs_dir2_samename(name, &xfs_name_dot)) { /* If this is "." then check that the inum matches the dir. */ - if (ino != dp->i_ino) + if (ino != I_INO(dp)) xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset); } else if (xfs_dir2_samename(name, &xfs_name_dotdot)) { /* * If this is ".." in the root inode, check that the inum * matches this dir. */ - if (xchk_inode_is_dirtree_root(dp) && ino != dp->i_ino) + if (xchk_inode_is_dirtree_root(dp) && ino != I_INO(dp)) xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset); } @@ -482,10 +482,10 @@ xchk_directory_data_bestfree( /* dir block format */ if (lblk != XFS_B_TO_FSBT(mp, XFS_DIR2_DATA_OFFSET)) xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, lblk); - error = xfs_dir3_block_read(sc->tp, sc->ip, sc->ip->i_ino, &bp); + error = xfs_dir3_block_read(sc->tp, sc->ip, I_INO(sc->ip), &bp); } else { /* dir data format */ - error = xfs_dir3_data_read(sc->tp, sc->ip, sc->ip->i_ino, lblk, + error = xfs_dir3_data_read(sc->tp, sc->ip, I_INO(sc->ip), lblk, 0, &bp); } if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, lblk, &error)) @@ -643,7 +643,7 @@ xchk_directory_leaf1_bestfree( int error; /* Read the free space block. */ - error = xfs_dir3_leaf_read(sc->tp, sc->ip, sc->ip->i_ino, lblk, &bp); + error = xfs_dir3_leaf_read(sc->tp, sc->ip, I_INO(sc->ip), lblk, &bp); if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, lblk, &error)) return error; xchk_buffer_recheck(sc, bp); @@ -749,7 +749,7 @@ xchk_directory_free_bestfree( int error; /* Read the free space block */ - error = xfs_dir2_free_read(sc->tp, sc->ip, sc->ip->i_ino, lblk, &bp); + error = xfs_dir2_free_read(sc->tp, sc->ip, I_INO(sc->ip), lblk, &bp); if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, lblk, &error)) return error; xchk_buffer_recheck(sc, bp); @@ -797,7 +797,7 @@ xchk_directory_blocks( .whichfork = XFS_DATA_FORK, .geo = sc->mp->m_dir_geo, .trans = sc->tp, - .owner = sc->ip->i_ino, + .owner = I_INO(sc->ip), }; struct xfs_ifork *ifp = xfs_ifork_ptr(sc->ip, XFS_DATA_FORK); struct xfs_mount *mp = sc->mp; @@ -996,7 +996,7 @@ xchk_dir_slow_dirent( */ lockmode = xchk_dir_lock_child(sc, ip); if (lockmode) { - trace_xchk_dir_slowpath(sc->ip, xname, ip->i_ino); + trace_xchk_dir_slowpath(sc->ip, xname, I_INO(ip)); goto check_pptr; } @@ -1007,7 +1007,7 @@ xchk_dir_slow_dirent( xchk_iunlock(sc, sc->ilock_flags); sd->need_revalidate = true; - trace_xchk_dir_ultraslowpath(sc->ip, xname, ip->i_ino); + trace_xchk_dir_ultraslowpath(sc->ip, xname, I_INO(ip)); error = xchk_dir_trylock_for_pptrs(sc, ip, &lockmode); if (error) @@ -1080,7 +1080,7 @@ xchk_directory( /* Plausible size? */ if (sc->ip->i_disk_size < xfs_dir2_sf_hdr_size(0)) { - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); return 0; } diff --git a/fs/xfs/scrub/dir_repair.c b/fs/xfs/scrub/dir_repair.c index 23b80c54aa60..1c088cfba10e 100644 --- a/fs/xfs/scrub/dir_repair.c +++ b/fs/xfs/scrub/dir_repair.c @@ -304,7 +304,7 @@ xrep_dir_want_salvage( struct xfs_mount *mp = rd->sc->mp; /* No pointers to ourselves or to garbage. */ - if (ino == rd->sc->ip->i_ino) + if (ino == I_INO(rd->sc->ip)) return false; if (!xfs_verify_dir_ino(mp, ino)) return false; @@ -644,14 +644,14 @@ xrep_dir_recover_dirblock( case cpu_to_be32(XFS_DIR3_BLOCK_MAGIC): if (!xrep_buf_verify_struct(bp, &xfs_dir3_block_buf_ops)) goto out; - if (xfs_dir3_block_header_check(bp, rd->sc->ip->i_ino) != NULL) + if (xfs_dir3_block_header_check(bp, I_INO(rd->sc->ip)) != NULL) goto out; break; case cpu_to_be32(XFS_DIR2_DATA_MAGIC): case cpu_to_be32(XFS_DIR3_DATA_MAGIC): if (!xrep_buf_verify_struct(bp, &xfs_dir3_data_buf_ops)) goto out; - if (xfs_dir3_data_header_check(bp, rd->sc->ip->i_ino) != NULL) + if (xfs_dir3_data_header_check(bp, I_INO(rd->sc->ip)) != NULL) goto out; break; default: @@ -676,7 +676,7 @@ xrep_dir_init_args( memset(&rd->args, 0, sizeof(struct xfs_da_args)); rd->args.geo = rd->sc->mp->m_dir_geo; rd->args.whichfork = XFS_DATA_FORK; - rd->args.owner = rd->sc->ip->i_ino; + rd->args.owner = I_INO(rd->sc->ip); rd->args.trans = rd->sc->tp; rd->args.dp = dp; if (!name) @@ -1125,12 +1125,12 @@ xrep_dir_scan_pptr( if (error) return error; - if (parent_ino != sc->ip->i_ino || + if (parent_ino != I_INO(sc->ip) || parent_gen != VFS_I(sc->ip)->i_generation) return 0; mutex_lock(&rd->pscan.lock); - error = xrep_dir_stash_createname(rd, &xname, ip->i_ino); + error = xrep_dir_stash_createname(rd, &xname, I_INO(ip)); mutex_unlock(&rd->pscan.lock); return error; } @@ -1151,7 +1151,7 @@ xrep_dir_scan_dirent( struct xrep_dir *rd = priv; /* Dirent doesn't point to this directory. */ - if (ino != rd->sc->ip->i_ino) + if (ino != I_INO(rd->sc->ip)) return 0; /* Ignore garbage inum. */ @@ -1168,9 +1168,9 @@ xrep_dir_scan_dirent( return 0; trace_xrep_dir_stash_createname(sc->tempip, &xfs_name_dotdot, - dp->i_ino); + I_INO(dp)); - xrep_findparent_scan_found(&rd->pscan, dp->i_ino); + xrep_findparent_scan_found(&rd->pscan, I_INO(dp)); return 0; } @@ -1281,7 +1281,7 @@ xrep_dir_scan_dirtree( /* Roots of directory trees are their own parents. */ if (xchk_inode_is_dirtree_root(sc->ip)) - xrep_findparent_scan_found(&rd->pscan, sc->ip->i_ino); + xrep_findparent_scan_found(&rd->pscan, I_INO(sc->ip)); /* * Filesystem scans are time consuming. Drop the directory ILOCK and @@ -1369,15 +1369,15 @@ xrep_dir_live_update( * rebuilding. Stash the update for replay against the temporary * directory. */ - if (p->dp->i_ino == sc->ip->i_ino && - xchk_iscan_want_live_update(&rd->pscan.iscan, p->ip->i_ino)) { + if (I_INO(p->dp) == I_INO(sc->ip) && + xchk_iscan_want_live_update(&rd->pscan.iscan, I_INO(p->ip))) { mutex_lock(&rd->pscan.lock); if (p->delta > 0) error = xrep_dir_stash_createname(rd, p->name, - p->ip->i_ino); + I_INO(p->ip)); else error = xrep_dir_stash_removename(rd, p->name, - p->ip->i_ino); + I_INO(p->ip)); mutex_unlock(&rd->pscan.lock); if (error) goto out_abort; @@ -1388,14 +1388,14 @@ xrep_dir_live_update( * the directory that we're rebuilding, so remember the new dotdot * target. */ - if (p->ip->i_ino == sc->ip->i_ino && - xchk_iscan_want_live_update(&rd->pscan.iscan, p->dp->i_ino)) { + if (I_INO(p->ip) == I_INO(sc->ip) && + xchk_iscan_want_live_update(&rd->pscan.iscan, I_INO(p->dp))) { if (p->delta > 0) { trace_xrep_dir_stash_createname(sc->tempip, &xfs_name_dotdot, - p->dp->i_ino); + I_INO(p->dp)); - xrep_findparent_scan_found(&rd->pscan, p->dp->i_ino); + xrep_findparent_scan_found(&rd->pscan, I_INO(p->dp)); } else { trace_xrep_dir_stash_removename(sc->tempip, &xfs_name_dotdot, @@ -1468,7 +1468,7 @@ xrep_dir_swap_prep( .whichfork = XFS_DATA_FORK, .trans = sc->tp, .total = 1, - .owner = sc->ip->i_ino, + .owner = I_INO(sc->ip), }; error = xfs_dir2_sf_to_block(&args); @@ -1576,7 +1576,7 @@ xrep_dir_set_nlink( * count. If the directory has no parent, it will be moved to the * orphanage. */ - pag = xfs_perag_get(sc->mp, XFS_INO_TO_AGNO(sc->mp, dp->i_ino)); + pag = xfs_perag_get(sc->mp, XFS_INODE_TO_AGNO(dp)); if (!pag) { ASSERT(0); return -EFSCORRUPTED; @@ -1764,7 +1764,7 @@ xrep_dir_rebuild_tree( * directory to an empty shortform directory because inactivation does * nothing for directories. */ - error = xrep_dir_reset_fork(rd, sc->mp->m_rootip->i_ino); + error = xrep_dir_reset_fork(rd, I_INO(sc->mp->m_rootip)); if (error) return error; diff --git a/fs/xfs/scrub/dirtree.c b/fs/xfs/scrub/dirtree.c index 143114718d75..c6a210f7508f 100644 --- a/fs/xfs/scrub/dirtree.c +++ b/fs/xfs/scrub/dirtree.c @@ -175,7 +175,7 @@ xchk_dirpath_append( if (error) return error; - error = xino_bitmap_set(&path->seen_inodes, ip->i_ino); + error = xino_bitmap_set(&path->seen_inodes, I_INO(ip)); if (error) return error; @@ -394,7 +394,7 @@ xchk_dirpath_step_up( * The inode being scanned is its own distant ancestor! Get rid of * this path. */ - if (parent_ino == sc->ip->i_ino) { + if (parent_ino == I_INO(sc->ip)) { xchk_dirpath_set_outcome(dl, path, XCHK_DIRPATH_DELETE); error = 0; goto out_scanlock; @@ -533,7 +533,7 @@ xchk_dirpath_walk_upwards( * The inode being scanned is its own direct ancestor! * Get rid of this path. */ - if (be64_to_cpu(dl->pptr_rec.p_ino) == sc->ip->i_ino) { + if (be64_to_cpu(dl->pptr_rec.p_ino) == I_INO(sc->ip)) { xchk_dirpath_set_outcome(dl, path, XCHK_DIRPATH_DELETE); return 0; } @@ -610,7 +610,7 @@ xchk_dirpath_step_is_stale( * If the parent and child being updated are not the ones mentioned in * this path step, the scan data is still ok. */ - if (p->ip->i_ino != child_ino || p->dp->i_ino != *cursor) + if (I_INO(p->ip) != child_ino || I_INO(p->dp) != *cursor) return 0; /* @@ -632,13 +632,13 @@ xchk_dirpath_step_is_stale( * If the update comes from the repair code itself, walk the state * machine forward. */ - if (p->ip->i_ino == dl->scan_ino && + if (I_INO(p->ip) == dl->scan_ino && path->outcome == XREP_DIRPATH_ADOPTING) { xchk_dirpath_set_outcome(dl, path, XREP_DIRPATH_ADOPTED); return 0; } - if (p->ip->i_ino == dl->scan_ino && + if (I_INO(p->ip) == dl->scan_ino && path->outcome == XREP_DIRPATH_DELETING) { xchk_dirpath_set_outcome(dl, path, XREP_DIRPATH_DELETED); return 0; @@ -669,7 +669,7 @@ xchk_dirpath_is_stale( * The child being updated has not been seen by this path at all; this * path cannot be stale. */ - if (!xino_bitmap_test(&path->seen_inodes, p->ip->i_ino)) + if (!xino_bitmap_test(&path->seen_inodes, I_INO(p->ip))) return 0; ret = xchk_dirpath_step_is_stale(dl, path, 0, idx, p, &cursor); @@ -928,7 +928,7 @@ xchk_dirtree( * released during teardown. */ dl->root_ino = xchk_inode_rootdir_inum(sc->ip); - dl->scan_ino = sc->ip->i_ino; + dl->scan_ino = I_INO(sc->ip); trace_xchk_dirtree_start(sc->ip, sc->sm, 0); @@ -954,7 +954,7 @@ xchk_dirtree( * parent pointers are corrupt; this scan cannot be completed * without full information. */ - xchk_ino_xref_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_xref_set_corrupt(sc, sc->ip); error = 0; goto out_scanlock; } @@ -979,12 +979,12 @@ xchk_dirtree( xchk_dirtree_evaluate(dl, &oc); if (xchk_dirtree_parentless(dl)) { if (oc.good || oc.bad || oc.suspect) - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); } else { if (oc.bad || oc.good + oc.suspect != 1) - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); if (oc.suspect) - xchk_ino_xref_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_xref_set_corrupt(sc, sc->ip); } out_scanlock: diff --git a/fs/xfs/scrub/dirtree_repair.c b/fs/xfs/scrub/dirtree_repair.c index bcfbbf5efc6c..1c0d7ea4a5be 100644 --- a/fs/xfs/scrub/dirtree_repair.c +++ b/fs/xfs/scrub/dirtree_repair.c @@ -459,7 +459,7 @@ xrep_dirtree_unlink( if (error) goto out_trans_cancel; - error = xfs_dir_removename(sc->tp, dp, &dl->xname, sc->ip->i_ino, + error = xfs_dir_removename(sc->tp, dp, &dl->xname, I_INO(sc->ip), resblks); if (error) { ASSERT(error != -ENOENT); @@ -583,7 +583,7 @@ xrep_dirtree_create_adoption_path( */ xfs_inode_to_parent_rec(&dl->pptr_rec, sc->orphanage); - error = xino_bitmap_set(&path->seen_inodes, sc->orphanage->i_ino); + error = xino_bitmap_set(&path->seen_inodes, I_INO(sc->orphanage)); if (error) goto out_path; diff --git a/fs/xfs/scrub/findparent.c b/fs/xfs/scrub/findparent.c index 2076f028d271..04b6b96b0a30 100644 --- a/fs/xfs/scrub/findparent.c +++ b/fs/xfs/scrub/findparent.c @@ -65,7 +65,7 @@ struct xrep_findparent_info { /* * Scrub context. We're looking for a @dp containing a directory - * entry pointing to sc->ip->i_ino. + * entry pointing to I_INO(sc->ip). */ struct xfs_scrub *sc; @@ -106,7 +106,7 @@ xrep_findparent_dirent( if (xchk_should_terminate(fpi->sc, &error)) return error; - if (ino != fpi->sc->ip->i_ino) + if (ino != I_INO(fpi->sc->ip)) return 0; /* Ignore garbage directory entry names. */ @@ -123,18 +123,18 @@ xrep_findparent_dirent( /* Uhoh, more than one parent for a dir? */ if (fpi->found_parent != NULLFSINO && - !(fpi->parent_tentative && fpi->found_parent == fpi->dp->i_ino)) { + !(fpi->parent_tentative && fpi->found_parent == I_INO(fpi->dp))) { trace_xrep_findparent_dirent(fpi->sc->ip, 0); return -EFSCORRUPTED; } /* We found a potential parent; remember this. */ - trace_xrep_findparent_dirent(fpi->sc->ip, fpi->dp->i_ino); - fpi->found_parent = fpi->dp->i_ino; + trace_xrep_findparent_dirent(fpi->sc->ip, I_INO(fpi->dp)); + fpi->found_parent = I_INO(fpi->dp); fpi->parent_tentative = false; if (fpi->parent_scan) - xrep_findparent_scan_found(fpi->parent_scan, fpi->dp->i_ino); + xrep_findparent_scan_found(fpi->parent_scan, I_INO(fpi->dp)); return 0; } @@ -227,13 +227,12 @@ xrep_findparent_live_update( * already scanned @p->dp, update the dotdot target inumber to the * parent inode. */ - if (p->ip->i_ino == sc->ip->i_ino && - xchk_iscan_want_live_update(&pscan->iscan, p->dp->i_ino)) { - if (p->delta > 0) { - xrep_findparent_scan_found(pscan, p->dp->i_ino); - } else { + if (I_INO(p->ip) == I_INO(sc->ip) && + xchk_iscan_want_live_update(&pscan->iscan, I_INO(p->dp))) { + if (p->delta > 0) + xrep_findparent_scan_found(pscan, I_INO(p->dp)); + else xrep_findparent_scan_found(pscan, NULLFSINO); - } } return NOTIFY_DONE; @@ -388,7 +387,7 @@ xrep_findparent_confirm( if (*parent_ino == NULLFSINO) return 0; if (!xfs_verify_dir_ino(sc->mp, *parent_ino) || - *parent_ino == sc->ip->i_ino) { + *parent_ino == I_INO(sc->ip)) { *parent_ino = NULLFSINO; return 0; } @@ -422,10 +421,10 @@ xfs_ino_t xrep_findparent_self_reference( struct xfs_scrub *sc) { - if (sc->ip->i_ino == sc->mp->m_sb.sb_rootino) + if (I_INO(sc->ip) == sc->mp->m_sb.sb_rootino) return sc->mp->m_sb.sb_rootino; - if (sc->ip->i_ino == sc->mp->m_sb.sb_metadirino) + if (I_INO(sc->ip) == sc->mp->m_sb.sb_metadirino) return sc->mp->m_sb.sb_metadirino; if (VFS_I(sc->ip)->i_nlink == 0) @@ -457,8 +456,8 @@ xrep_findparent_from_dcache( dput(parent); if (S_ISDIR(pip->i_mode)) { - trace_xrep_findparent_from_dcache(sc->ip, XFS_I(pip)->i_ino); - ret = XFS_I(pip)->i_ino; + ret = pip->i_ino; + trace_xrep_findparent_from_dcache(sc->ip, ret); } xchk_irele(sc, XFS_I(pip)); diff --git a/fs/xfs/scrub/ialloc.c b/fs/xfs/scrub/ialloc.c index 911dc0f9a79d..19c0b1b2a787 100644 --- a/fs/xfs/scrub/ialloc.c +++ b/fs/xfs/scrub/ialloc.c @@ -363,6 +363,7 @@ xchk_iallocbt_check_cluster( struct xfs_inobt_rec_incore *irec, unsigned int cluster_base) { + struct xfs_perag *pag = to_perag(bs->cur->bc_group); struct xfs_imap imap; struct xfs_mount *mp = bs->cur->bc_mp; struct xfs_buf *cluster_bp; @@ -394,8 +395,7 @@ xchk_iallocbt_check_cluster( * ir_startino can be large enough to make im_boffset nonzero. */ ir_holemask = (irec->ir_holemask & cluster_mask); - imap.im_blkno = xfs_agbno_to_daddr(to_perag(bs->cur->bc_group), agbno); - imap.im_len = XFS_FSB_TO_BB(mp, M_IGEO(mp)->blocks_per_cluster); + imap.im_agbno = agbno; imap.im_boffset = XFS_INO_TO_OFFSET(mp, irec->ir_startino) << mp->m_sb.sb_inodelog; @@ -405,8 +405,8 @@ xchk_iallocbt_check_cluster( return 0; } - trace_xchk_iallocbt_check_cluster(to_perag(bs->cur->bc_group), - irec->ir_startino, imap.im_blkno, imap.im_len, + trace_xchk_iallocbt_check_cluster(pag, irec->ir_startino, imap.im_agbno, + XFS_FSB_TO_BB(mp, M_IGEO(mp)->blocks_per_cluster), cluster_base, nr_inodes, cluster_mask, ir_holemask, XFS_INO_TO_OFFSET(mp, irec->ir_startino + cluster_base)); @@ -429,7 +429,8 @@ xchk_iallocbt_check_cluster( &XFS_RMAP_OINFO_INODES); /* Grab the inode cluster buffer. */ - error = xfs_imap_to_bp(mp, bs->cur->bc_tp, &imap, &cluster_bp); + error = xfs_read_icluster(pag, bs->cur->bc_tp, imap.im_agbno, + &cluster_bp); if (!xchk_btree_xref_process_error(bs->sc, bs->cur, 0, &error)) return error; diff --git a/fs/xfs/scrub/ialloc_repair.c b/fs/xfs/scrub/ialloc_repair.c index 608c7022ac15..46b1c8ac5543 100644 --- a/fs/xfs/scrub/ialloc_repair.c +++ b/fs/xfs/scrub/ialloc_repair.c @@ -287,7 +287,6 @@ xrep_ibt_process_cluster( struct xrep_ibt *ri, xfs_agblock_t cluster_bno) { - struct xfs_imap imap; struct xfs_buf *cluster_bp; struct xfs_scrub *sc = ri->sc; struct xfs_mount *mp = sc->mp; @@ -305,10 +304,8 @@ xrep_ibt_process_cluster( * inobt because imap_to_bp directly maps the buffer without touching * either inode btree. */ - imap.im_blkno = xfs_agbno_to_daddr(sc->sa.pag, cluster_bno); - imap.im_len = XFS_FSB_TO_BB(mp, igeo->blocks_per_cluster); - imap.im_boffset = 0; - error = xfs_imap_to_bp(mp, sc->tp, &imap, &cluster_bp); + error = xfs_read_icluster(sc->sa.pag, sc->tp, cluster_bno, + &cluster_bp); if (error) return error; diff --git a/fs/xfs/scrub/inode.c b/fs/xfs/scrub/inode.c index 948d04dcba2a..65b13e311916 100644 --- a/fs/xfs/scrub/inode.c +++ b/fs/xfs/scrub/inode.c @@ -102,7 +102,7 @@ xchk_setup_inode( xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN); /* We want to scan the opened inode, so lock it and exit. */ - if (sc->sm->sm_ino == 0 || sc->sm->sm_ino == ip_in->i_ino) { + if (sc->sm->sm_ino == 0 || sc->sm->sm_ino == I_INO(ip_in)) { error = xchk_install_live_inode(sc, ip_in); if (error) return error; @@ -710,17 +710,17 @@ xchk_inode_xref_bmap( if (!xchk_should_check_xref(sc, &error, NULL)) return; if (nextents < xfs_dfork_data_extents(dip)) - xchk_ino_xref_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_xref_set_corrupt(sc, sc->ip); error = xchk_inode_count_blocks(sc, XFS_ATTR_FORK, &nextents, &acount); if (!xchk_should_check_xref(sc, &error, NULL)) return; if (nextents != xfs_dfork_attr_extents(dip)) - xchk_ino_xref_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_xref_set_corrupt(sc, sc->ip); /* Check nblocks against the inode. */ if (count + acount != be64_to_cpu(dip->di_nblocks)) - xchk_ino_xref_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_xref_set_corrupt(sc, sc->ip); } /* Cross-reference with the other btrees. */ @@ -794,10 +794,10 @@ xchk_inode_check_unlinked( { if (VFS_I(sc->ip)->i_nlink == 0) { if (!xfs_inode_on_unlinked_list(sc->ip)) - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ino_set_corrupt(sc, I_INO(sc->ip)); } else { if (xfs_inode_on_unlinked_list(sc->ip)) - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ino_set_corrupt(sc, I_INO(sc->ip)); } } @@ -821,7 +821,7 @@ xchk_inode( /* Scrub the inode core. */ xfs_inode_to_disk(sc->ip, &di, 0); - xchk_dinode(sc, &di, sc->ip->i_ino); + xchk_dinode(sc, &di, I_INO(sc->ip)); if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) goto out; @@ -831,11 +831,11 @@ xchk_inode( * we scrubbed the dinode. */ if (S_ISREG(VFS_I(sc->ip)->i_mode)) - xchk_inode_check_reflink_iflag(sc, sc->ip->i_ino); + xchk_inode_check_reflink_iflag(sc, I_INO(sc->ip)); xchk_inode_check_unlinked(sc); - xchk_inode_xref(sc, sc->ip->i_ino, &di); + xchk_inode_xref(sc, I_INO(sc->ip), &di); out: return error; } diff --git a/fs/xfs/scrub/inode_repair.c b/fs/xfs/scrub/inode_repair.c index 9738b9ce3f2d..493dcf5cc6c1 100644 --- a/fs/xfs/scrub/inode_repair.c +++ b/fs/xfs/scrub/inode_repair.c @@ -1547,6 +1547,7 @@ xrep_dinode_core( struct xrep_inode *ri) { struct xfs_scrub *sc = ri->sc; + struct xfs_mount *mp = sc->mp; struct xfs_buf *bp; struct xfs_dinode *dip; xfs_ino_t ino = sc->sm->sm_ino; @@ -1559,8 +1560,11 @@ xrep_dinode_core( return error; /* Read the inode cluster buffer. */ - error = xfs_trans_read_buf(sc->mp, sc->tp, sc->mp->m_ddev_targp, - ri->imap.im_blkno, ri->imap.im_len, 0, &bp, NULL); + error = xfs_trans_read_buf(mp, sc->tp, mp->m_ddev_targp, + XFS_AGB_TO_DADDR(mp, XFS_INO_TO_AGNO(mp, ino), + ri->imap.im_agbno), + XFS_FSB_TO_BB(mp, M_IGEO(mp)->blocks_per_cluster), + 0, &bp, NULL); if (error) return error; @@ -1583,10 +1587,10 @@ xrep_dinode_core( write: /* Write out the inode. */ trace_xrep_dinode_fixed(sc, dip); - xfs_dinode_calc_crc(sc->mp, dip); + xfs_dinode_calc_crc(mp, dip); xfs_trans_buf_set_type(sc->tp, bp, XFS_BLFT_DINO_BUF); xfs_trans_log_buf(sc->tp, bp, ri->imap.im_boffset, - ri->imap.im_boffset + sc->mp->m_sb.sb_inodesize - 1); + ri->imap.im_boffset + mp->m_sb.sb_inodesize - 1); /* * In theory, we've fixed the ondisk inode record enough that we should @@ -1796,7 +1800,7 @@ xrep_inode_flags( sc->ip->i_diflags &= ~XFS_DIFLAG_ANY; /* NEWRTBM only applies to realtime bitmaps */ - if (sc->ip->i_ino == sc->mp->m_sb.sb_rbmino) + if (I_INO(sc->ip) == sc->mp->m_sb.sb_rbmino) sc->ip->i_diflags |= XFS_DIFLAG_NEWRTBM; else sc->ip->i_diflags &= ~XFS_DIFLAG_NEWRTBM; @@ -2011,8 +2015,7 @@ xrep_inode_unlinked( struct xfs_perag *pag; int error; - pag = xfs_perag_get(sc->mp, - XFS_INO_TO_AGNO(sc->mp, sc->ip->i_ino)); + pag = xfs_perag_get(sc->mp, XFS_INODE_TO_AGNO(sc->ip)); error = xfs_iunlink_remove(sc->tp, pag, sc->ip); xfs_perag_put(pag); if (error) diff --git a/fs/xfs/scrub/iscan.c b/fs/xfs/scrub/iscan.c index 2a974eed00cc..03f72ba9e7a3 100644 --- a/fs/xfs/scrub/iscan.c +++ b/fs/xfs/scrub/iscan.c @@ -728,7 +728,7 @@ xchk_iscan_mark_visited( struct xfs_inode *ip) { mutex_lock(&iscan->lock); - iscan->__visited_ino = ip->i_ino; + iscan->__visited_ino = I_INO(ip); trace_xchk_iscan_visit(iscan); mutex_unlock(&iscan->lock); } diff --git a/fs/xfs/scrub/listxattr.c b/fs/xfs/scrub/listxattr.c index 0863db64b1b2..9b4cf393788b 100644 --- a/fs/xfs/scrub/listxattr.c +++ b/fs/xfs/scrub/listxattr.c @@ -115,7 +115,7 @@ xchk_xattr_walk_leaf( struct xfs_buf *leaf_bp; int error; - error = xfs_attr3_leaf_read(sc->tp, ip, ip->i_ino, 0, &leaf_bp); + error = xfs_attr3_leaf_read(sc->tp, ip, I_INO(ip), 0, &leaf_bp); if (error) return error; @@ -166,7 +166,7 @@ xchk_xattr_find_leftmost_leaf( magic != XFS_DA3_NODE_MAGIC) goto out_buf; - fa = xfs_da3_node_header_check(bp, ip->i_ino); + fa = xfs_da3_node_header_check(bp, I_INO(ip)); if (fa) goto out_buf; @@ -195,7 +195,7 @@ xchk_xattr_find_leftmost_leaf( } error = -EFSCORRUPTED; - fa = xfs_attr3_leaf_header_check(bp, ip->i_ino); + fa = xfs_attr3_leaf_header_check(bp, I_INO(ip)); if (fa) goto out_buf; @@ -266,8 +266,8 @@ xchk_xattr_walk_node( goto out_bitmap; } - error = xfs_attr3_leaf_read(sc->tp, ip, ip->i_ino, - leafhdr.forw, &leaf_bp); + error = xfs_attr3_leaf_read(sc->tp, ip, I_INO(ip), leafhdr.forw, + &leaf_bp); if (error) goto out_bitmap; diff --git a/fs/xfs/scrub/metapath.c b/fs/xfs/scrub/metapath.c index 050b86eb12d3..ff1ff762b300 100644 --- a/fs/xfs/scrub/metapath.c +++ b/fs/xfs/scrub/metapath.c @@ -314,7 +314,7 @@ xchk_metapath( /* Parent required to do anything else. */ if (mpath->dp == NULL) { - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); return 0; } @@ -329,15 +329,15 @@ xchk_metapath( trace_xchk_metapath_lookup(sc, mpath->path, mpath->dp, ino); if (error == -ENOENT) { /* No directory entry at all */ - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); error = 0; goto out_ilock; } if (!xchk_fblock_xref_process_error(sc, XFS_DATA_FORK, 0, &error)) goto out_ilock; - if (ino != sc->ip->i_ino) { + if (ino != I_INO(sc->ip)) { /* Pointing to wrong inode */ - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); } out_ilock: @@ -364,7 +364,7 @@ xrep_metapath_link( else mpath->du.ppargs = NULL; - trace_xrep_metapath_link(sc, mpath->path, mpath->dp, sc->ip->i_ino); + trace_xrep_metapath_link(sc, mpath->path, mpath->dp, I_INO(sc->ip)); return xfs_dir_add_child(sc->tp, mpath->link_resblks, &mpath->du); } @@ -462,7 +462,7 @@ xrep_metapath_try_link( if (error) goto out_cancel; - if (ino == sc->ip->i_ino) { + if (ino == I_INO(sc->ip)) { /* The dirent already points to @sc->ip; we're done. */ error = 0; goto out_cancel; @@ -530,7 +530,7 @@ xrep_metapath_try_unlink( xfs_ino_t ino; int error; - ASSERT(*alleged_child != sc->ip->i_ino); + ASSERT(*alleged_child != I_INO(sc->ip)); trace_xrep_metapath_try_unlink(sc, mpath->path, mpath->dp, *alleged_child); @@ -575,7 +575,7 @@ xrep_metapath_try_unlink( if (error) goto out_cancel; - if (ino == sc->ip->i_ino) { + if (ino == I_INO(sc->ip)) { /* The dirent already points to @sc->ip; we're done. */ error = -EEXIST; goto out_cancel; diff --git a/fs/xfs/scrub/newbt.c b/fs/xfs/scrub/newbt.c index 2e4981c1baf2..c82f4631fd9c 100644 --- a/fs/xfs/scrub/newbt.c +++ b/fs/xfs/scrub/newbt.c @@ -123,8 +123,7 @@ xrep_newbt_init_inode( if (!ifp) return -ENOMEM; - xrep_newbt_init_ag(xnr, sc, oinfo, - XFS_INO_TO_FSB(sc->mp, sc->ip->i_ino), + xrep_newbt_init_ag(xnr, sc, oinfo, XFS_INODE_TO_FSB(sc->ip), XFS_AG_RESV_NONE); xnr->ifake.if_fork = ifp; xnr->ifake.if_fork_size = xfs_inode_fork_size(sc->ip, whichfork); @@ -146,7 +145,7 @@ xrep_newbt_init_metadir_inode( ASSERT(xfs_is_metadir_inode(sc->ip)); - xfs_rmap_ino_bmbt_owner(&oinfo, sc->ip->i_ino, XFS_DATA_FORK); + xfs_rmap_inode_bmbt_owner(&oinfo, sc->ip, XFS_DATA_FORK); ifp = kmem_cache_zalloc(xfs_ifork_cache, XCHK_GFP_FLAGS); if (!ifp) @@ -160,8 +159,7 @@ xrep_newbt_init_metadir_inode( * as if they were regular file blocks. This exposes us to a higher * risk of the repair being cancelled due to ENOSPC. */ - xrep_newbt_init_ag(xnr, sc, &oinfo, - XFS_INO_TO_FSB(sc->mp, sc->ip->i_ino), + xrep_newbt_init_ag(xnr, sc, &oinfo, XFS_INODE_TO_FSB(sc->ip), XFS_AG_RESV_NONE); xnr->ifake.if_fork = ifp; xnr->ifake.if_fork_size = xfs_inode_fork_size(sc->ip, XFS_DATA_FORK); diff --git a/fs/xfs/scrub/nlinks.c b/fs/xfs/scrub/nlinks.c index 1e0effdb65a6..355ab6de23ea 100644 --- a/fs/xfs/scrub/nlinks.c +++ b/fs/xfs/scrub/nlinks.c @@ -175,7 +175,7 @@ xchk_nlinks_live_update( if (xrep_is_tempfile(p->dp)) return NOTIFY_DONE; - trace_xchk_nlinks_live_update(xnc->sc->mp, p->dp, action, p->ip->i_ino, + trace_xchk_nlinks_live_update(xnc->sc->mp, p->dp, action, I_INO(p->ip), p->delta, p->name->name, p->name->len); /* @@ -183,12 +183,12 @@ xchk_nlinks_live_update( * to @ip. If @ip is a subdirectory, update the number of child links * going out of @dp. */ - if (xchk_iscan_want_live_update(&xnc->collect_iscan, p->dp->i_ino)) { + if (xchk_iscan_want_live_update(&xnc->collect_iscan, I_INO(p->dp))) { mutex_lock(&xnc->lock); - error = xchk_nlinks_update_incore(xnc, p->ip->i_ino, p->delta, + error = xchk_nlinks_update_incore(xnc, I_INO(p->ip), p->delta, 0, 0); if (!error && S_ISDIR(VFS_IC(p->ip)->i_mode)) - error = xchk_nlinks_update_incore(xnc, p->dp->i_ino, 0, + error = xchk_nlinks_update_incore(xnc, I_INO(p->dp), 0, 0, p->delta); mutex_unlock(&xnc->lock); if (error) @@ -200,9 +200,9 @@ xchk_nlinks_live_update( * number of backrefs pointing to @dp. */ if (S_ISDIR(VFS_IC(p->ip)->i_mode) && - xchk_iscan_want_live_update(&xnc->collect_iscan, p->ip->i_ino)) { + xchk_iscan_want_live_update(&xnc->collect_iscan, I_INO(p->ip))) { mutex_lock(&xnc->lock); - error = xchk_nlinks_update_incore(xnc, p->dp->i_ino, 0, + error = xchk_nlinks_update_incore(xnc, I_INO(p->dp), 0, p->delta, 0); mutex_unlock(&xnc->lock); if (error) @@ -243,7 +243,7 @@ xchk_nlinks_collect_dirent( dotdot = true; /* Don't accept a '.' entry that points somewhere else. */ - if (dot && ino != dp->i_ino) { + if (dot && ino != I_INO(dp)) { error = -ECANCELED; goto out_abort; } @@ -304,7 +304,7 @@ xchk_nlinks_collect_dirent( * number of child links of dp. */ if (!dot && !dotdot && name->type == XFS_DIR3_FT_DIR) { - error = xchk_nlinks_update_incore(xnc, dp->i_ino, 0, 0, 1); + error = xchk_nlinks_update_incore(xnc, I_INO(dp), 0, 0, 1); if (error) goto out_unlock; } @@ -692,7 +692,7 @@ xchk_nlinks_compare_inode( goto out_scanlock; } - error = xchk_nlinks_comparison_read(xnc, ip->i_ino, &obs); + error = xchk_nlinks_comparison_read(xnc, I_INO(ip), &obs); if (error) goto out_scanlock; @@ -718,15 +718,15 @@ xchk_nlinks_compare_inode( * count, but it will let them decrease it. */ if (total_links > XFS_NLINK_PINNED) { - xchk_ino_set_corrupt(sc, ip->i_ino); + xchk_ip_set_corrupt(sc, ip); goto out_corrupt; } else if (total_links > XFS_MAXLINK) { - xchk_ino_set_warning(sc, ip->i_ino); + xchk_ino_set_warning(sc, I_INO(ip)); } /* Link counts should match. */ if (total_links != actual_nlink) { - xchk_ino_set_corrupt(sc, ip->i_ino); + xchk_ip_set_corrupt(sc, ip); goto out_corrupt; } @@ -740,14 +740,14 @@ xchk_nlinks_compare_inode( * number of subdirectory entries in the directory. */ if (obs.children != obs.backrefs) - xchk_ino_xref_set_corrupt(sc, ip->i_ino); + xchk_ip_xref_set_corrupt(sc, ip); } else { /* * Non-directories and unlinked directories should not have * back references. */ if (obs.backrefs != 0) { - xchk_ino_set_corrupt(sc, ip->i_ino); + xchk_ip_set_corrupt(sc, ip); goto out_corrupt; } @@ -756,7 +756,7 @@ xchk_nlinks_compare_inode( * children. */ if (obs.children != 0) { - xchk_ino_set_corrupt(sc, ip->i_ino); + xchk_ip_set_corrupt(sc, ip); goto out_corrupt; } } @@ -769,7 +769,7 @@ xchk_nlinks_compare_inode( * the root directory. */ if (obs.parents != 1) { - xchk_ino_set_corrupt(sc, ip->i_ino); + xchk_ip_set_corrupt(sc, ip); goto out_corrupt; } } else if (actual_nlink > 0) { @@ -778,7 +778,7 @@ xchk_nlinks_compare_inode( * least one parent. */ if (obs.parents == 0) { - xchk_ino_set_corrupt(sc, ip->i_ino); + xchk_ip_set_corrupt(sc, ip); goto out_corrupt; } } diff --git a/fs/xfs/scrub/nlinks_repair.c b/fs/xfs/scrub/nlinks_repair.c index 9049215c6eae..fbc2ff809fc0 100644 --- a/fs/xfs/scrub/nlinks_repair.c +++ b/fs/xfs/scrub/nlinks_repair.c @@ -75,7 +75,7 @@ xrep_nlinks_iunlink_remove( struct xfs_perag *pag; int error; - pag = xfs_perag_get(sc->mp, XFS_INO_TO_AGNO(sc->mp, sc->ip->i_ino)); + pag = xfs_perag_get(sc->mp, XFS_INODE_TO_AGNO(sc->ip)); error = xfs_iunlink_remove(sc->tp, pag, sc->ip); xfs_perag_put(pag); return error; @@ -152,7 +152,7 @@ xrep_nlinks_repair_inode( goto out_scanlock; } - error = xfarray_load_sparse(xnc->nlinks, ip->i_ino, &obs); + error = xfarray_load_sparse(xnc->nlinks, I_INO(ip), &obs); if (error) goto out_scanlock; @@ -206,7 +206,7 @@ xrep_nlinks_repair_inode( * updated our scan info. */ mutex_lock(&xnc->lock); - error = xfarray_load_sparse(xnc->nlinks, ip->i_ino, &obs); + error = xfarray_load_sparse(xnc->nlinks, I_INO(ip), &obs); mutex_unlock(&xnc->lock); if (error) goto out_trans; diff --git a/fs/xfs/scrub/orphanage.c b/fs/xfs/scrub/orphanage.c index 33c6db6b4498..3aca66869b80 100644 --- a/fs/xfs/scrub/orphanage.c +++ b/fs/xfs/scrub/orphanage.c @@ -402,14 +402,14 @@ xrep_adoption_compute_name( int error = 0; adopt->xname = xname; - xname->len = snprintf(namebuf, MAXNAMELEN, "%llu", sc->ip->i_ino); + xname->len = snprintf(namebuf, MAXNAMELEN, "%llu", I_INO(sc->ip)); xname->type = xfs_mode_to_ftype(VFS_I(sc->ip)->i_mode); /* Make sure the filename is unique in the lost+found. */ error = xchk_dir_lookup(sc, sc->orphanage, xname, &ino); while (error == 0 && incr < 10000) { xname->len = snprintf(namebuf, MAXNAMELEN, "%llu.%u", - sc->ip->i_ino, ++incr); + I_INO(sc->ip), ++incr); error = xchk_dir_lookup(sc, sc->orphanage, xname, &ino); } if (error == 0) { @@ -532,7 +532,7 @@ xrep_adoption_move( int error; trace_xrep_adoption_reparent(sc->orphanage, adopt->xname, - sc->ip->i_ino); + I_INO(sc->ip)); error = xrep_adoption_check_dcache(adopt); if (error) @@ -553,7 +553,7 @@ xrep_adoption_move( /* Create the new name in the orphanage. */ error = xfs_dir_createname(sc->tp, sc->orphanage, adopt->xname, - sc->ip->i_ino, adopt->orphanage_blkres); + I_INO(sc->ip), adopt->orphanage_blkres); if (error) return error; @@ -576,7 +576,7 @@ xrep_adoption_move( /* Replace the dotdot entry if the child is a subdirectory. */ if (isdir) { error = xfs_dir_replace(sc->tp, sc->ip, &xfs_name_dotdot, - sc->orphanage->i_ino, adopt->child_blkres); + I_INO(sc->orphanage), adopt->child_blkres); if (error) return error; } diff --git a/fs/xfs/scrub/parent.c b/fs/xfs/scrub/parent.c index 931604fa1294..a8c4807e1d94 100644 --- a/fs/xfs/scrub/parent.c +++ b/fs/xfs/scrub/parent.c @@ -72,7 +72,7 @@ xchk_parent_actor( if (!xchk_fblock_xref_process_error(sc, XFS_DATA_FORK, 0, &error)) return error; - if (sc->ip->i_ino == ino) + if (I_INO(sc->ip) == ino) spc->nlink++; if (xchk_should_terminate(spc->sc, &error)) @@ -119,6 +119,7 @@ xchk_parent_validate( .nlink = 0, }; struct xfs_mount *mp = sc->mp; + xfs_ino_t ino = I_INO(sc->ip); struct xfs_inode *dp = NULL; xfs_nlink_t expected_nlink; unsigned int lock_mode; @@ -126,22 +127,20 @@ xchk_parent_validate( /* Is this the root dir? Then '..' must point to itself. */ if (sc->ip == mp->m_rootip) { - if (sc->ip->i_ino != mp->m_sb.sb_rootino || - sc->ip->i_ino != parent_ino) + if (ino != mp->m_sb.sb_rootino || ino != parent_ino) xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); return 0; } /* Is this the metadata root dir? Then '..' must point to itself. */ if (sc->ip == mp->m_metadirip) { - if (sc->ip->i_ino != mp->m_sb.sb_metadirino || - sc->ip->i_ino != parent_ino) + if (ino != mp->m_sb.sb_metadirino || ino != parent_ino) xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); return 0; } /* '..' must not point to ourselves. */ - if (sc->ip->i_ino == parent_ino) { + if (ino == parent_ino) { xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); return 0; } @@ -315,7 +314,7 @@ xchk_parent_pptr_and_dotdot( /* Is this the root dir? Then '..' must point to itself. */ if (xchk_inode_is_dirtree_root(sc->ip)) { - if (sc->ip->i_ino != pp->parent_ino) + if (I_INO(sc->ip) != pp->parent_ino) xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); return 0; } @@ -398,7 +397,7 @@ xchk_parent_dirent( return error; /* Does the inode number match? */ - if (child_ino != sc->ip->i_ino) { + if (child_ino != I_INO(sc->ip)) { xchk_fblock_xref_set_corrupt(sc, XFS_ATTR_FORK, 0); return 0; } @@ -490,7 +489,7 @@ xchk_parent_scan_attr( } /* No self-referential parent pointers. */ - if (parent_ino == sc->ip->i_ino) { + if (parent_ino == I_INO(sc->ip)) { xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, 0); return -ECANCELED; } @@ -512,7 +511,7 @@ xchk_parent_scan_attr( }; /* Couldn't lock the inode, so save the pptr for later. */ - trace_xchk_parent_defer(sc->ip, &xname, dp->i_ino); + trace_xchk_parent_defer(sc->ip, &xname, I_INO(dp)); error = xfblob_storename(pp->pptr_names, &save_pp.name_cookie, &xname); @@ -599,7 +598,7 @@ xchk_parent_slow_pptr( */ lockmode = xchk_parent_lock_dir(sc, dp); if (lockmode) { - trace_xchk_parent_slowpath(sc->ip, xname, dp->i_ino); + trace_xchk_parent_slowpath(sc->ip, xname, I_INO(dp)); goto check_dirent; } @@ -610,7 +609,7 @@ xchk_parent_slow_pptr( xchk_iunlock(sc, sc->ilock_flags); pp->need_revalidate = true; - trace_xchk_parent_ultraslowpath(sc->ip, xname, dp->i_ino); + trace_xchk_parent_ultraslowpath(sc->ip, xname, I_INO(dp)); error = xchk_dir_trylock_for_pptrs(sc, dp, &lockmode); if (error) @@ -729,10 +728,10 @@ xchk_parent_count_pptrs( pp->pptrs_found++; if (VFS_I(sc->ip)->i_nlink == 0 && pp->pptrs_found > 0) - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); else if (VFS_I(sc->ip)->i_nlink > 0 && pp->pptrs_found == 0) - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); } else { /* * Starting with metadir, we allow checking of parent pointers @@ -743,7 +742,7 @@ xchk_parent_count_pptrs( pp->pptrs_found++; if (VFS_I(sc->ip)->i_nlink != pp->pptrs_found) - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); } return 0; @@ -856,7 +855,7 @@ xchk_parent( return -ENOENT; /* We're not a special inode, are we? */ - if (!xfs_verify_dir_ino(mp, sc->ip->i_ino)) { + if (!xfs_verify_dir_ino(mp, I_INO(sc->ip))) { xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); return 0; } diff --git a/fs/xfs/scrub/parent_repair.c b/fs/xfs/scrub/parent_repair.c index 2ce0cefad362..f643724910f9 100644 --- a/fs/xfs/scrub/parent_repair.c +++ b/fs/xfs/scrub/parent_repair.c @@ -302,14 +302,14 @@ xrep_parent_replay_update( trace_xrep_parent_replay_parentadd(sc->tempip, xname, &pptr->pptr_rec); - return xfs_parent_set(sc->tempip, sc->ip->i_ino, xname, + return xfs_parent_set(sc->tempip, I_INO(sc->ip), xname, &pptr->pptr_rec, &rp->pptr_args); case XREP_PPTR_REMOVE: /* Remove parent pointer. */ trace_xrep_parent_replay_parentremove(sc->tempip, xname, &pptr->pptr_rec); - return xfs_parent_unset(sc->tempip, sc->ip->i_ino, xname, + return xfs_parent_unset(sc->tempip, I_INO(sc->ip), xname, &pptr->pptr_rec, &rp->pptr_args); } @@ -434,7 +434,7 @@ xrep_parent_scan_dirent( int error; /* Dirent doesn't point to this directory. */ - if (ino != rp->sc->ip->i_ino) + if (ino != I_INO(rp->sc->ip)) return 0; /* No weird looking names. */ @@ -645,8 +645,8 @@ xrep_parent_live_update( * repairing, so stash the update for replay against the temporary * file. */ - if (p->ip->i_ino == sc->ip->i_ino && - xchk_iscan_want_live_update(&rp->pscan.iscan, p->dp->i_ino)) { + if (I_INO(p->ip) == I_INO(sc->ip) && + xchk_iscan_want_live_update(&rp->pscan.iscan, I_INO(p->dp))) { mutex_lock(&rp->pscan.lock); if (p->delta > 0) error = xrep_parent_stash_parentadd(rp, p->name, p->dp); @@ -906,7 +906,7 @@ xrep_parent_fetch_xattr_remote( .namelen = namelen, .trans = sc->tp, .valuelen = valuelen, - .owner = ip->i_ino, + .owner = I_INO(ip), }; int error; @@ -984,7 +984,7 @@ xrep_parent_insert_xattr( .attr_filter = key->flags, .namelen = key->namelen, .valuelen = key->valuelen, - .owner = rp->sc->ip->i_ino, + .owner = I_INO(rp->sc->ip), .geo = rp->sc->mp->m_attr_geo, .whichfork = XFS_ATTR_FORK, .op_flags = XFS_DA_OP_OKNOENT, @@ -1329,7 +1329,7 @@ xrep_parent_rebuild_pptrs( * For this purpose, root directories are their own parents. */ if (xchk_inode_is_dirtree_root(sc->ip)) { - xrep_findparent_scan_found(&rp->pscan, sc->ip->i_ino); + xrep_findparent_scan_found(&rp->pscan, I_INO(sc->ip)); } else { error = xrep_parent_lookup_pptrs(sc, &parent_ino); if (error) @@ -1451,7 +1451,7 @@ xrep_parent_set_nondir_nlink( * The file is on the unlinked list but we found parents. * Remove the file from the unlinked list. */ - pag = xfs_perag_get(sc->mp, XFS_INO_TO_AGNO(sc->mp, ip->i_ino)); + pag = xfs_perag_get(sc->mp, XFS_INODE_TO_AGNO(ip)); if (!pag) { ASSERT(0); return -EFSCORRUPTED; diff --git a/fs/xfs/scrub/quotacheck.c b/fs/xfs/scrub/quotacheck.c index 9ea014e2ecfd..c199d128538e 100644 --- a/fs/xfs/scrub/quotacheck.c +++ b/fs/xfs/scrub/quotacheck.c @@ -399,7 +399,7 @@ xqcheck_collect_inode( int error = 0; if (xfs_is_metadir_inode(ip) || - xfs_is_quota_inode(&tp->t_mountp->m_sb, ip->i_ino)) { + xfs_is_quota_inode(&tp->t_mountp->m_sb, I_INO(ip))) { /* * Quota files are never counted towards quota, so we do not * need to take the lock. Files do not switch between the diff --git a/fs/xfs/scrub/readdir.c b/fs/xfs/scrub/readdir.c index c66ec9093a38..d68bf20b4a14 100644 --- a/fs/xfs/scrub/readdir.c +++ b/fs/xfs/scrub/readdir.c @@ -50,7 +50,7 @@ xchk_dir_walk_sf( dapos = xfs_dir2_db_off_to_dataptr(geo, geo->datablk, geo->data_entry_offset); - error = dirent_fn(sc, dp, dapos, &name, dp->i_ino, priv); + error = dirent_fn(sc, dp, dapos, &name, I_INO(dp), priv); if (error) return error; @@ -100,7 +100,7 @@ xchk_dir_walk_block( unsigned int off, next_off, end; int error; - error = xfs_dir3_block_read(sc->tp, dp, dp->i_ino, &bp); + error = xfs_dir3_block_read(sc->tp, dp, I_INO(dp), &bp); if (error) return error; @@ -176,7 +176,7 @@ xchk_read_leaf_dir_buf( if (new_off > *curoff) *curoff = new_off; - return xfs_dir3_data_read(tp, dp, dp->i_ino, map.br_startoff, 0, bpp); + return xfs_dir3_data_read(tp, dp, I_INO(dp), map.br_startoff, 0, bpp); } /* Call a function for every entry in a leaf directory. */ @@ -274,7 +274,7 @@ xchk_dir_walk( .dp = dp, .geo = dp->i_mount->m_dir_geo, .trans = sc->tp, - .owner = dp->i_ino, + .owner = I_INO(dp), }; int error; @@ -320,7 +320,7 @@ xchk_dir_lookup( .hashval = xfs_dir2_hashname(dp->i_mount, name), .whichfork = XFS_DATA_FORK, .op_flags = XFS_DA_OP_OKNOENT, - .owner = dp->i_ino, + .owner = I_INO(dp), }; int error; @@ -332,7 +332,7 @@ xchk_dir_lookup( * set to sc->ip, so we must switch the owner here for the lookup. */ if (dp == sc->tempip) - args.owner = sc->ip->i_ino; + args.owner = I_INO(sc->ip); ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); xfs_assert_ilocked(dp, XFS_ILOCK_SHARED | XFS_ILOCK_EXCL); diff --git a/fs/xfs/scrub/reap.c b/fs/xfs/scrub/reap.c index fff23932828b..fcd14c1703ea 100644 --- a/fs/xfs/scrub/reap.c +++ b/fs/xfs/scrub/reap.c @@ -1205,7 +1205,7 @@ xrep_reap_metadir_fsblocks( ASSERT(xfs_is_metadir_inode(sc->ip)); xreap_configure_agextent_limits(&rs); - xfs_rmap_ino_bmbt_owner(&oinfo, sc->ip->i_ino, XFS_DATA_FORK); + xfs_rmap_inode_bmbt_owner(&oinfo, sc->ip, XFS_DATA_FORK); error = xfsb_bitmap_walk(bitmap, xreap_fsmeta_extent, &rs); if (error) return error; @@ -1249,8 +1249,7 @@ xreap_bmapi_select( cur = xfs_rmapbt_init_cursor(sc->mp, sc->tp, sc->sa.agf_bp, sc->sa.pag); - xfs_rmap_ino_owner(&oinfo, rs->ip->i_ino, rs->whichfork, - imap->br_startoff); + xfs_rmap_inode_owner(&oinfo, rs->ip, rs->whichfork, imap->br_startoff); error = xfs_rmap_has_other_keys(cur, agbno, 1, &oinfo, crosslinked); if (error) goto out_cur; diff --git a/fs/xfs/scrub/repair.c b/fs/xfs/scrub/repair.c index ac8c592579bd..11697a8b2a1d 100644 --- a/fs/xfs/scrub/repair.c +++ b/fs/xfs/scrub/repair.c @@ -816,7 +816,7 @@ xrep_ino_dqattach( case -ENOENT: xfs_err_ratelimited(sc->mp, "inode %llu repair encountered quota error %d, quotacheck forced.", - (unsigned long long)sc->ip->i_ino, error); + (unsigned long long)I_INO(sc->ip), error); if (XFS_IS_UQUOTA_ON(sc->mp) && !sc->ip->i_udquot) xrep_force_quotacheck(sc, XFS_DQTYPE_USER); if (XFS_IS_GQUOTA_ON(sc->mp) && !sc->ip->i_gdquot) diff --git a/fs/xfs/scrub/rmap_repair.c b/fs/xfs/scrub/rmap_repair.c index 6766df9922e8..590f9f41856e 100644 --- a/fs/xfs/scrub/rmap_repair.c +++ b/fs/xfs/scrub/rmap_repair.c @@ -570,7 +570,7 @@ xrep_rmap_scan_ifork( int whichfork) { struct xrep_rmap_ifork rf = { - .accum = { .rm_owner = ip->i_ino, }, + .accum = { .rm_owner = I_INO(ip), }, .rr = rr, .whichfork = whichfork, }; diff --git a/fs/xfs/scrub/rtbitmap.c b/fs/xfs/scrub/rtbitmap.c index b3a4972d423e..de3f22f310f7 100644 --- a/fs/xfs/scrub/rtbitmap.c +++ b/fs/xfs/scrub/rtbitmap.c @@ -200,13 +200,13 @@ xchk_rtbitmap( /* Is sb_rextents correct? */ if (mp->m_sb.sb_rextents != rtb->rextents) { - xchk_ino_set_corrupt(sc, rbmip->i_ino); + xchk_ip_set_corrupt(sc, rbmip); return 0; } /* Is sb_rextslog correct? */ if (mp->m_sb.sb_rextslog != rtb->rextslog) { - xchk_ino_set_corrupt(sc, rbmip->i_ino); + xchk_ip_set_corrupt(sc, rbmip); return 0; } @@ -215,17 +215,17 @@ xchk_rtbitmap( * case can we exceed 4bn bitmap blocks since the super field is a u32. */ if (rtb->rbmblocks > U32_MAX) { - xchk_ino_set_corrupt(sc, rbmip->i_ino); + xchk_ip_set_corrupt(sc, rbmip); return 0; } if (mp->m_sb.sb_rbmblocks != rtb->rbmblocks) { - xchk_ino_set_corrupt(sc, rbmip->i_ino); + xchk_ip_set_corrupt(sc, rbmip); return 0; } /* The bitmap file length must be aligned to an fsblock. */ if (rbmip->i_disk_size & mp->m_blockmask) { - xchk_ino_set_corrupt(sc, rbmip->i_ino); + xchk_ip_set_corrupt(sc, rbmip); return 0; } @@ -235,7 +235,7 @@ xchk_rtbitmap( * file can be larger than sb_rbmblocks. */ if (rbmip->i_disk_size < XFS_FSB_TO_B(mp, rtb->rbmblocks)) { - xchk_ino_set_corrupt(sc, rbmip->i_ino); + xchk_ip_set_corrupt(sc, rbmip); return 0; } @@ -284,7 +284,7 @@ xchk_xref_is_used_rt_space( if (xfs_has_zoned(sc->mp)) { if (!xfs_zone_rgbno_is_valid(rtg, xfs_rtb_to_rgbno(sc->mp, rtbno) + len - 1)) - xchk_ino_xref_set_corrupt(sc, rtg_rmap(rtg)->i_ino); + xchk_ip_xref_set_corrupt(sc, rtg_rmap(rtg)); return; } @@ -295,5 +295,5 @@ xchk_xref_is_used_rt_space( if (!xchk_should_check_xref(sc, &error, NULL)) return; if (is_free) - xchk_ino_xref_set_corrupt(sc, rtg_bitmap(rtg)->i_ino); + xchk_ip_xref_set_corrupt(sc, rtg_bitmap(rtg)); } diff --git a/fs/xfs/scrub/rtbitmap_repair.c b/fs/xfs/scrub/rtbitmap_repair.c index f4ca86a2ea1b..dc64902d6c25 100644 --- a/fs/xfs/scrub/rtbitmap_repair.c +++ b/fs/xfs/scrub/rtbitmap_repair.c @@ -382,7 +382,7 @@ xrep_rtbitmap_prep_buf( struct xfs_rtbuf_blkinfo *hdr = bp->b_addr; hdr->rt_magic = cpu_to_be32(XFS_RTBITMAP_MAGIC); - hdr->rt_owner = cpu_to_be64(sc->ip->i_ino); + hdr->rt_owner = cpu_to_be64(I_INO(sc->ip)); hdr->rt_blkno = cpu_to_be64(xfs_buf_daddr(bp)); hdr->rt_lsn = 0; uuid_copy(&hdr->rt_uuid, &sc->mp->m_sb.sb_meta_uuid); diff --git a/fs/xfs/scrub/rtrefcount.c b/fs/xfs/scrub/rtrefcount.c index 4951cc271788..0d10ce2910c2 100644 --- a/fs/xfs/scrub/rtrefcount.c +++ b/fs/xfs/scrub/rtrefcount.c @@ -555,7 +555,7 @@ xchk_rtrefcountbt( if (error || (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) return error; - xfs_rmap_ino_bmbt_owner(&btree_oinfo, rtg_refcount(sc->sr.rtg)->i_ino, + xfs_rmap_inode_bmbt_owner(&btree_oinfo, rtg_refcount(sc->sr.rtg), XFS_DATA_FORK); error = xchk_btree(sc, sc->sr.refc_cur, xchk_rtrefcountbt_rec, &btree_oinfo, &rrc); diff --git a/fs/xfs/scrub/rtrefcount_repair.c b/fs/xfs/scrub/rtrefcount_repair.c index f165fb397647..2b939960c7dd 100644 --- a/fs/xfs/scrub/rtrefcount_repair.c +++ b/fs/xfs/scrub/rtrefcount_repair.c @@ -360,7 +360,7 @@ xrep_rtrefc_walk_rmap( return error; /* Skip extents which are not owned by this inode and fork. */ - if (rec->rm_owner != rr->sc->ip->i_ino) + if (rec->rm_owner != I_INO(rr->sc->ip)) return 0; error = xrep_check_ino_btree_mapping(rr->sc, rec); diff --git a/fs/xfs/scrub/rtrmap.c b/fs/xfs/scrub/rtrmap.c index 8b1a8389d32f..043be93c7148 100644 --- a/fs/xfs/scrub/rtrmap.c +++ b/fs/xfs/scrub/rtrmap.c @@ -244,7 +244,6 @@ int xchk_rtrmapbt( struct xfs_scrub *sc) { - struct xfs_inode *ip = rtg_rmap(sc->sr.rtg); struct xfs_owner_info oinfo; struct xchk_rtrmap cr = { }; int error; @@ -253,7 +252,7 @@ xchk_rtrmapbt( if (error || (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) return error; - xfs_rmap_ino_bmbt_owner(&oinfo, ip->i_ino, XFS_DATA_FORK); + xfs_rmap_inode_bmbt_owner(&oinfo, rtg_rmap(sc->sr.rtg), XFS_DATA_FORK); return xchk_btree(sc, sc->sr.rmap_cur, xchk_rtrmapbt_rec, &oinfo, &cr); } diff --git a/fs/xfs/scrub/rtrmap_repair.c b/fs/xfs/scrub/rtrmap_repair.c index f2d16ca8d251..a2b72e61edf5 100644 --- a/fs/xfs/scrub/rtrmap_repair.c +++ b/fs/xfs/scrub/rtrmap_repair.c @@ -324,7 +324,7 @@ xrep_rtrmap_scan_dfork( struct xfs_inode *ip) { struct xrep_rtrmap_ifork rf = { - .accum = { .rm_owner = ip->i_ino, }, + .accum = { .rm_owner = I_INO(ip), }, .rr = rr, }; struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_DATA_FORK); @@ -392,7 +392,7 @@ xrep_rtrmap_walk_rmap( return error; /* Skip extents which are not owned by this inode and fork. */ - if (rec->rm_owner != rr->sc->ip->i_ino) + if (rec->rm_owner != I_INO(rr->sc->ip)) return 0; error = xrep_check_ino_btree_mapping(rr->sc, rec); diff --git a/fs/xfs/scrub/rtsummary.c b/fs/xfs/scrub/rtsummary.c index efce39d7a1ef..78f72a046887 100644 --- a/fs/xfs/scrub/rtsummary.c +++ b/fs/xfs/scrub/rtsummary.c @@ -189,7 +189,7 @@ xchk_rtsum_record_free( rtlen = xfs_rtxlen_to_extlen(mp, rec->ar_extcount); if (!xfs_verify_rtbext(mp, rtbno, rtlen)) { - xchk_ino_xref_set_corrupt(sc, rtg_bitmap(rtg)->i_ino); + xchk_ip_xref_set_corrupt(sc, rtg_bitmap(rtg)); return -EFSCORRUPTED; } @@ -314,25 +314,25 @@ xchk_rtsummary( /* Is sb_rextents correct? */ if (mp->m_sb.sb_rextents != rts->rextents) { - xchk_ino_set_corrupt(sc, rbmip->i_ino); + xchk_ip_set_corrupt(sc, rbmip); return 0; } /* Is m_rsumlevels correct? */ if (mp->m_rsumlevels != rts->rsumlevels) { - xchk_ino_set_corrupt(sc, rsumip->i_ino); + xchk_ip_set_corrupt(sc, rsumip); return 0; } /* Is m_rsumsize correct? */ if (mp->m_rsumblocks != rts->rsumblocks) { - xchk_ino_set_corrupt(sc, rsumip->i_ino); + xchk_ip_set_corrupt(sc, rsumip); return 0; } /* The summary file length must be aligned to an fsblock. */ if (rsumip->i_disk_size & mp->m_blockmask) { - xchk_ino_set_corrupt(sc, rsumip->i_ino); + xchk_ip_set_corrupt(sc, rsumip); return 0; } @@ -342,7 +342,7 @@ xchk_rtsummary( * the file can be larger than rsumsize. */ if (rsumip->i_disk_size < XFS_FSB_TO_B(mp, rts->rsumblocks)) { - xchk_ino_set_corrupt(sc, rsumip->i_ino); + xchk_ip_set_corrupt(sc, rsumip); return 0; } @@ -358,7 +358,7 @@ xchk_rtsummary( * EFSCORRUPTED means the rtbitmap is corrupt, which is an xref * error since we're checking the summary file. */ - xchk_ino_set_corrupt(sc, rbmip->i_ino); + xchk_ip_set_corrupt(sc, rbmip); return 0; } if (error) diff --git a/fs/xfs/scrub/rtsummary_repair.c b/fs/xfs/scrub/rtsummary_repair.c index afffbd6e0ad1..f065c3e51ce2 100644 --- a/fs/xfs/scrub/rtsummary_repair.c +++ b/fs/xfs/scrub/rtsummary_repair.c @@ -91,7 +91,7 @@ xrep_rtsummary_prep_buf( struct xfs_rtbuf_blkinfo *hdr = bp->b_addr; hdr->rt_magic = cpu_to_be32(XFS_RTSUMMARY_MAGIC); - hdr->rt_owner = cpu_to_be64(sc->ip->i_ino); + hdr->rt_owner = cpu_to_be64(I_INO(sc->ip)); hdr->rt_blkno = cpu_to_be64(xfs_buf_daddr(bp)); hdr->rt_lsn = 0; uuid_copy(&hdr->rt_uuid, &sc->mp->m_sb.sb_meta_uuid); diff --git a/fs/xfs/scrub/scrub.c b/fs/xfs/scrub/scrub.c index 27511a125b39..8742445c86f4 100644 --- a/fs/xfs/scrub/scrub.c +++ b/fs/xfs/scrub/scrub.c @@ -955,7 +955,7 @@ xfs_ioc_scrubv_metadata( * because each scrubber gets to decide its own strategy and return * values for getting an inode. */ - if (head.svh_ino && head.svh_ino != ip_in->i_ino) + if (head.svh_ino && head.svh_ino != I_INO(ip_in)) handle_ip = xchk_scrubv_open_by_handle(mp, &head); /* Run all the scrubbers. */ diff --git a/fs/xfs/scrub/symlink_repair.c b/fs/xfs/scrub/symlink_repair.c index 25416dfb5189..91c86ea0e0f1 100644 --- a/fs/xfs/scrub/symlink_repair.c +++ b/fs/xfs/scrub/symlink_repair.c @@ -144,7 +144,7 @@ xrep_symlink_salvage_remote( fa = bp->b_ops->verify_struct(bp); dsl = bp->b_addr; magic_ok = dsl->sl_magic == cpu_to_be32(XFS_SYMLINK_MAGIC); - hdr_ok = xfs_symlink_hdr_ok(ip->i_ino, offset, byte_cnt, bp); + hdr_ok = xfs_symlink_hdr_ok(I_INO(ip), offset, byte_cnt, bp); if (!hdr_ok || (fa != NULL && !magic_ok)) break; @@ -259,7 +259,7 @@ xrep_symlink_local_to_remote( if (!xfs_has_crc(sc->mp)) return; - dsl->sl_owner = cpu_to_be64(sc->ip->i_ino); + dsl->sl_owner = cpu_to_be64(I_INO(sc->ip)); xfs_trans_log_buf(tp, bp, 0, sizeof(struct xfs_dsymlink_hdr) + ifp->if_bytes - 1); } @@ -375,7 +375,7 @@ xrep_symlink_reset_fork( /* Reset the temp symlink target to dummy content. */ xfs_idestroy_fork(ifp); - return xfs_symlink_write_target(sc->tp, sc->tempip, sc->tempip->i_ino, + return xfs_symlink_write_target(sc->tp, sc->tempip, I_INO(sc->tempip), "?", 1, 0, 0); } @@ -434,7 +434,7 @@ xrep_symlink_rebuild( sc->tempip->i_df.if_format = XFS_DINODE_FMT_EXTENTS; /* Write the salvaged target to the temporary link. */ - error = xfs_symlink_write_target(sc->tp, sc->tempip, sc->ip->i_ino, + error = xfs_symlink_write_target(sc->tp, sc->tempip, I_INO(sc->ip), target_buf, target_len, fs_blocks, resblks); if (error) return error; diff --git a/fs/xfs/scrub/tempfile.c b/fs/xfs/scrub/tempfile.c index 8d754df72aa5..e0c630f888cf 100644 --- a/fs/xfs/scrub/tempfile.c +++ b/fs/xfs/scrub/tempfile.c @@ -121,7 +121,7 @@ xrep_tempfile_create( * remote target block, so the owner is irrelevant. */ error = xfs_symlink_write_target(tp, sc->tempip, - sc->tempip->i_ino, ".", 1, 0, 0); + I_INO(sc->tempip), ".", 1, 0, 0); if (error) goto out_trans_cancel; } diff --git a/fs/xfs/scrub/trace.c b/fs/xfs/scrub/trace.c index 70d353287993..e5efc680651f 100644 --- a/fs/xfs/scrub/trace.c +++ b/fs/xfs/scrub/trace.c @@ -48,7 +48,7 @@ xchk_btree_cur_fsbno( if (level == cur->bc_nlevels - 1 && cur->bc_ops->type == XFS_BTREE_TYPE_INODE) - return XFS_INO_TO_FSB(cur->bc_mp, cur->bc_ino.ip->i_ino); + return XFS_INODE_TO_FSB(cur->bc_ino.ip); return NULLFSBLOCK; } diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index 286c5f5e0544..1b7d9e07a27d 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -162,7 +162,7 @@ DECLARE_EVENT_CLASS(xchk_class, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->type = sm->sm_type; __entry->agno = sm->sm_agno; __entry->inum = sm->sm_ino; @@ -236,7 +236,7 @@ DECLARE_EVENT_CLASS(xchk_vector_head_class, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->agno = vhead->svh_agno; __entry->inum = vhead->svh_ino; __entry->gen = vhead->svh_gen; @@ -340,7 +340,7 @@ TRACE_EVENT(xchk_file_op_error, ), TP_fast_assign( __entry->dev = sc->ip->i_mount->m_super->s_dev; - __entry->ino = sc->ip->i_ino; + __entry->ino = I_INO(sc->ip); __entry->whichfork = whichfork; __entry->type = sc->sm->sm_type; __entry->offset = offset; @@ -438,7 +438,7 @@ DECLARE_EVENT_CLASS(xchk_fblock_error_class, ), TP_fast_assign( __entry->dev = sc->ip->i_mount->m_super->s_dev; - __entry->ino = sc->ip->i_ino; + __entry->ino = I_INO(sc->ip); __entry->whichfork = whichfork; __entry->type = sc->sm->sm_type; __entry->offset = offset; @@ -481,7 +481,7 @@ DECLARE_EVENT_CLASS(xchk_dqiter_class, TP_fast_assign( __entry->dev = cursor->sc->mp->m_super->s_dev; __entry->dqtype = cursor->dqtype; - __entry->ino = cursor->quota_ip->i_ino; + __entry->ino = I_INO(cursor->quota_ip); __entry->cur_id = cursor->id; __entry->startoff = cursor->bmap.br_startoff; __entry->startblock = cursor->bmap.br_startblock; @@ -613,7 +613,7 @@ TRACE_EVENT(xchk_ifork_btree_op_error, TP_fast_assign( xfs_fsblock_t fsbno = xchk_btree_cur_fsbno(cur, level); __entry->dev = sc->mp->m_super->s_dev; - __entry->ino = cur->bc_ino.ip->i_ino; + __entry->ino = I_INO(cur->bc_ino.ip); __entry->whichfork = cur->bc_ino.whichfork; __entry->type = sc->sm->sm_type; __assign_str(name); @@ -693,7 +693,7 @@ TRACE_EVENT(xchk_ifork_btree_error, TP_fast_assign( xfs_fsblock_t fsbno = xchk_btree_cur_fsbno(cur, level); __entry->dev = sc->mp->m_super->s_dev; - __entry->ino = sc->ip->i_ino; + __entry->ino = I_INO(sc->ip); __entry->whichfork = cur->bc_ino.whichfork; __entry->type = sc->sm->sm_type; __assign_str(name); @@ -785,17 +785,17 @@ TRACE_EVENT(xchk_xref_error, TRACE_EVENT(xchk_iallocbt_check_cluster, TP_PROTO(const struct xfs_perag *pag, xfs_agino_t startino, - xfs_daddr_t map_daddr, unsigned short map_len, + xfs_agblock_t map_agblock, unsigned short map_len, unsigned int chunk_ino, unsigned int nr_inodes, uint16_t cluster_mask, uint16_t holemask, unsigned int cluster_ino), - TP_ARGS(pag, startino, map_daddr, map_len, chunk_ino, nr_inodes, + TP_ARGS(pag, startino, map_agblock, map_len, chunk_ino, nr_inodes, cluster_mask, holemask, cluster_ino), TP_STRUCT__entry( __field(dev_t, dev) __field(xfs_agnumber_t, agno) __field(xfs_agino_t, startino) - __field(xfs_daddr_t, map_daddr) + __field(xfs_agblock_t, map_agblock) __field(unsigned short, map_len) __field(unsigned int, chunk_ino) __field(unsigned int, nr_inodes) @@ -807,7 +807,7 @@ TRACE_EVENT(xchk_iallocbt_check_cluster, __entry->dev = pag_mount(pag)->m_super->s_dev; __entry->agno = pag_agno(pag); __entry->startino = startino; - __entry->map_daddr = map_daddr; + __entry->map_agblock = map_agblock; __entry->map_len = map_len; __entry->chunk_ino = chunk_ino; __entry->nr_inodes = nr_inodes; @@ -815,11 +815,11 @@ TRACE_EVENT(xchk_iallocbt_check_cluster, __entry->holemask = holemask; __entry->cluster_ino = cluster_ino; ), - TP_printk("dev %d:%d agno 0x%x startino 0x%x daddr 0x%llx bbcount 0x%x chunkino 0x%x nr_inodes %u cluster_mask 0x%x holemask 0x%x cluster_ino 0x%x", + TP_printk("dev %d:%d agno 0x%x startino 0x%x agblock 0x%x bbcount 0x%x chunkino 0x%x nr_inodes %u cluster_mask 0x%x holemask 0x%x cluster_ino 0x%x", MAJOR(__entry->dev), MINOR(__entry->dev), __entry->agno, __entry->startino, - __entry->map_daddr, + __entry->map_agblock, __entry->map_len, __entry->chunk_ino, __entry->nr_inodes, @@ -839,7 +839,7 @@ TRACE_EVENT(xchk_inode_is_allocated, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->iflags = ip->i_flags; __entry->mode = VFS_I(ip)->i_mode; ), @@ -1437,7 +1437,7 @@ TRACE_EVENT(xchk_nlinks_collect_dirent, ), TP_fast_assign( __entry->dev = mp->m_super->s_dev; - __entry->dir = dp->i_ino; + __entry->dir = I_INO(dp); __entry->ino = ino; __entry->namelen = name->len; memcpy(__get_str(name), name->name, name->len); @@ -1464,7 +1464,7 @@ TRACE_EVENT(xchk_nlinks_collect_pptr, ), TP_fast_assign( __entry->dev = mp->m_super->s_dev; - __entry->dir = dp->i_ino; + __entry->dir = I_INO(dp); __entry->ino = be64_to_cpu(pptr->p_ino); __entry->namelen = name->len; memcpy(__get_str(name), name->name, name->len); @@ -1509,7 +1509,7 @@ TRACE_EVENT(xchk_nlinks_live_update, ), TP_fast_assign( __entry->dev = mp->m_super->s_dev; - __entry->dir = dp ? dp->i_ino : NULLFSINO; + __entry->dir = dp ? I_INO(dp) : NULLFSINO; __entry->action = action; __entry->ino = ino; __entry->delta = delta; @@ -1602,7 +1602,7 @@ DECLARE_EVENT_CLASS(xchk_nlinks_diff_class, ), TP_fast_assign( __entry->dev = mp->m_super->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->ftype = xfs_mode_to_ftype(VFS_I(ip)->i_mode); __entry->nlink = VFS_I(ip)->i_nlink; __entry->parents = live->parents; @@ -1638,7 +1638,7 @@ DECLARE_EVENT_CLASS(xchk_pptr_class, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->namelen = name->len; memcpy(__get_str(name), name, name->len); __entry->far_ino = far_ino; @@ -1680,7 +1680,7 @@ DECLARE_EVENT_CLASS(xchk_dirtree_class, TP_fast_assign( __entry->dev = sc->mp->m_super->s_dev; __entry->path_nr = path_nr; - __entry->child_ino = ip->i_ino; + __entry->child_ino = I_INO(ip); __entry->child_gen = VFS_I(ip)->i_generation; __entry->parent_ino = be64_to_cpu(pptr->p_ino); __entry->parent_gen = be32_to_cpu(pptr->p_gen); @@ -1727,7 +1727,7 @@ DECLARE_EVENT_CLASS(xchk_dirpath_class, __entry->dev = sc->mp->m_super->s_dev; __entry->path_nr = path_nr; __entry->step_nr = step_nr; - __entry->child_ino = ip->i_ino; + __entry->child_ino = I_INO(ip); __entry->child_gen = VFS_I(ip)->i_generation; __entry->parent_ino = be64_to_cpu(pptr->p_ino); __entry->parent_gen = be32_to_cpu(pptr->p_gen); @@ -1830,7 +1830,7 @@ DECLARE_EVENT_CLASS(xchk_dirtree_evaluate_class, ), TP_fast_assign( __entry->dev = dl->sc->mp->m_super->s_dev; - __entry->ino = dl->sc->ip->i_ino; + __entry->ino = I_INO(dl->sc->ip); __entry->rootino = dl->root_ino; __entry->nr_paths = dl->nr_paths; __entry->bad = oc->bad; @@ -1873,8 +1873,8 @@ TRACE_EVENT(xchk_dirpath_changed, __entry->dev = sc->mp->m_super->s_dev; __entry->path_nr = path_nr; __entry->step_nr = step_nr; - __entry->child_ino = ip->i_ino; - __entry->parent_ino = dp->i_ino; + __entry->child_ino = I_INO(ip); + __entry->parent_ino = I_INO(dp); __entry->namelen = xname->len; memcpy(__get_str(name), xname->name, xname->len); ), @@ -1904,9 +1904,9 @@ TRACE_EVENT(xchk_dirtree_live_update, ), TP_fast_assign( __entry->dev = sc->mp->m_super->s_dev; - __entry->parent_ino = dp->i_ino; + __entry->parent_ino = I_INO(dp); __entry->action = action; - __entry->child_ino = ip->i_ino; + __entry->child_ino = I_INO(ip); __entry->delta = delta; __entry->namelen = xname->len; memcpy(__get_str(name), xname->name, xname->len); @@ -1933,8 +1933,8 @@ DECLARE_EVENT_CLASS(xchk_metapath_class, ), TP_fast_assign( __entry->dev = sc->mp->m_super->s_dev; - __entry->scrub_ino = sc->ip ? sc->ip->i_ino : NULLFSINO; - __entry->parent_ino = dp ? dp->i_ino : NULLFSINO; + __entry->scrub_ino = sc->ip ? I_INO(sc->ip) : NULLFSINO; + __entry->parent_ino = dp ? I_INO(dp) : NULLFSINO; __entry->ino = ino; __assign_str(name); ), @@ -2208,7 +2208,7 @@ TRACE_EVENT(xrep_bmap_found, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->whichfork = whichfork; __entry->lblk = irec->br_startoff; __entry->len = irec->br_blockcount; @@ -2609,7 +2609,7 @@ TRACE_EVENT(xrep_dinode_findmode_dirent, TP_fast_assign( __entry->dev = sc->mp->m_super->s_dev; __entry->ino = sc->sm->sm_ino; - __entry->parent_ino = dp->i_ino; + __entry->parent_ino = I_INO(dp); __entry->ftype = ftype; ), TP_printk("dev %d:%d ino 0x%llx parent_ino 0x%llx ftype '%s'", @@ -2633,7 +2633,7 @@ TRACE_EVENT(xrep_dinode_findmode_dirent_inval, TP_fast_assign( __entry->dev = sc->mp->m_super->s_dev; __entry->ino = sc->sm->sm_ino; - __entry->parent_ino = dp->i_ino; + __entry->parent_ino = I_INO(dp); __entry->ftype = ftype; __entry->found_ftype = found_ftype; ), @@ -2658,7 +2658,7 @@ TRACE_EVENT(xrep_cow_mark_file_range, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->startoff = startoff; __entry->startblock = startblock; __entry->blockcount = blockcount; @@ -2687,7 +2687,7 @@ TRACE_EVENT(xrep_cow_replace_mapping, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->startoff = irec->br_startoff; __entry->startblock = irec->br_startblock; __entry->blockcount = irec->br_blockcount; @@ -2817,13 +2817,13 @@ TRACE_EVENT(xrep_tempfile_create, ), TP_fast_assign( __entry->dev = sc->mp->m_super->s_dev; - __entry->ino = sc->file ? XFS_I(file_inode(sc->file))->i_ino : 0; + __entry->ino = sc->file ? file_inode(sc->file)->i_ino : 0; __entry->type = sc->sm->sm_type; __entry->agno = sc->sm->sm_agno; __entry->inum = sc->sm->sm_ino; __entry->gen = sc->sm->sm_gen; __entry->flags = sc->sm->sm_flags; - __entry->temp_inum = sc->tempip->i_ino; + __entry->temp_inum = I_INO(sc->tempip); ), TP_printk("dev %d:%d ino 0x%llx type %s inum 0x%llx gen 0x%x flags 0x%x temp_inum 0x%llx", MAJOR(__entry->dev), MINOR(__entry->dev), @@ -2850,7 +2850,7 @@ DECLARE_EVENT_CLASS(xrep_tempfile_class, ), TP_fast_assign( __entry->dev = sc->mp->m_super->s_dev; - __entry->ino = sc->tempip->i_ino; + __entry->ino = I_INO(sc->tempip); __entry->whichfork = whichfork; __entry->lblk = irec->br_startoff; __entry->len = irec->br_blockcount; @@ -2890,7 +2890,7 @@ TRACE_EVENT(xreap_ifork_extent, ), TP_fast_assign( __entry->dev = sc->mp->m_super->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->whichfork = whichfork; __entry->fileoff = irec->br_startoff; __entry->len = irec->br_blockcount; @@ -2946,7 +2946,7 @@ TRACE_EVENT(xrep_xattr_recover_leafblock, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->dabno = dabno; __entry->magic = magic; ), @@ -2971,7 +2971,7 @@ DECLARE_EVENT_CLASS(xrep_xattr_salvage_class, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->flags = flags; __entry->namelen = namelen; memcpy(__get_str(name), name, namelen); @@ -3011,7 +3011,7 @@ DECLARE_EVENT_CLASS(xrep_pptr_salvage_class, const struct xfs_parent_rec *rec = value; __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->parent_ino = be64_to_cpu(rec->p_ino); __entry->parent_gen = be32_to_cpu(rec->p_gen); __entry->namelen = namelen; @@ -3043,8 +3043,8 @@ DECLARE_EVENT_CLASS(xrep_xattr_class, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; - __entry->src_ino = arg_ip->i_ino; + __entry->ino = I_INO(ip); + __entry->src_ino = I_INO(arg_ip); ), TP_printk("dev %d:%d ino 0x%llx src 0x%llx", MAJOR(__entry->dev), MINOR(__entry->dev), @@ -3073,8 +3073,8 @@ DECLARE_EVENT_CLASS(xrep_xattr_pptr_scan_class, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; - __entry->parent_ino = dp->i_ino; + __entry->ino = I_INO(ip); + __entry->parent_ino = I_INO(dp); __entry->parent_gen = VFS_IC(dp)->i_generation; __entry->namelen = name->len; memcpy(__get_str(name), name->name, name->len); @@ -3108,7 +3108,7 @@ TRACE_EVENT(xrep_dir_recover_dirblock, ), TP_fast_assign( __entry->dev = dp->i_mount->m_super->s_dev; - __entry->dir_ino = dp->i_ino; + __entry->dir_ino = I_INO(dp); __entry->dabno = dabno; __entry->magic = magic; __entry->magic_guess = magic_guess; @@ -3131,7 +3131,7 @@ DECLARE_EVENT_CLASS(xrep_dir_class, ), TP_fast_assign( __entry->dev = dp->i_mount->m_super->s_dev; - __entry->dir_ino = dp->i_ino; + __entry->dir_ino = I_INO(dp); __entry->parent_ino = parent_ino; ), TP_printk("dev %d:%d dir 0x%llx parent 0x%llx", @@ -3161,7 +3161,7 @@ DECLARE_EVENT_CLASS(xrep_dirent_class, ), TP_fast_assign( __entry->dev = dp->i_mount->m_super->s_dev; - __entry->dir_ino = dp->i_ino; + __entry->dir_ino = I_INO(dp); __entry->namelen = name->len; memcpy(__get_str(name), name->name, name->len); __entry->ino = ino; @@ -3198,8 +3198,8 @@ DECLARE_EVENT_CLASS(xrep_adoption_class, ), TP_fast_assign( __entry->dev = dp->i_mount->m_super->s_dev; - __entry->dir_ino = dp->i_ino; - __entry->child_ino = ip->i_ino; + __entry->dir_ino = I_INO(dp); + __entry->child_ino = I_INO(ip); __entry->moved = moved; ), TP_printk("dev %d:%d dir 0x%llx child 0x%llx moved? %d", @@ -3224,7 +3224,7 @@ DECLARE_EVENT_CLASS(xrep_parent_salvage_class, ), TP_fast_assign( __entry->dev = dp->i_mount->m_super->s_dev; - __entry->dir_ino = dp->i_ino; + __entry->dir_ino = I_INO(dp); __entry->ino = ino; ), TP_printk("dev %d:%d dir 0x%llx parent 0x%llx", @@ -3254,7 +3254,7 @@ DECLARE_EVENT_CLASS(xrep_pptr_class, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->parent_ino = be64_to_cpu(pptr->p_ino); __entry->parent_gen = be32_to_cpu(pptr->p_gen); __entry->namelen = name->len; @@ -3292,8 +3292,8 @@ DECLARE_EVENT_CLASS(xrep_pptr_scan_class, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; - __entry->parent_ino = dp->i_ino; + __entry->ino = I_INO(ip); + __entry->parent_ino = I_INO(dp); __entry->parent_gen = VFS_IC(dp)->i_generation; __entry->namelen = name->len; memcpy(__get_str(name), name->name, name->len); @@ -3392,7 +3392,7 @@ TRACE_EVENT(xrep_symlink_salvage_target, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->targetlen = targetlen; memcpy(__get_str(target), target, targetlen); __get_str(target)[targetlen] = 0; @@ -3413,7 +3413,7 @@ DECLARE_EVENT_CLASS(xrep_symlink_class, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); ), TP_printk("dev %d:%d ip 0x%llx", MAJOR(__entry->dev), MINOR(__entry->dev), @@ -3443,7 +3443,7 @@ TRACE_EVENT(xrep_iunlink_visit, TP_fast_assign( __entry->dev = pag_mount(pag)->m_super->s_dev; __entry->agno = pag_agno(pag); - __entry->agino = XFS_INO_TO_AGINO(pag_mount(pag), ip->i_ino); + __entry->agino = XFS_INODE_TO_AGINO(ip); __entry->bucket = bucket; __entry->bucket_agino = bucket_agino; __entry->prev_agino = ip->i_prev_unlinked; @@ -3473,8 +3473,8 @@ TRACE_EVENT(xrep_iunlink_reload_next, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->agno = XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino); - __entry->agino = XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino); + __entry->agno = XFS_INODE_TO_AGNO(ip); + __entry->agino = XFS_INODE_TO_AGINO(ip); __entry->old_prev_agino = ip->i_prev_unlinked; __entry->prev_agino = prev_agino; __entry->next_agino = ip->i_next_unlinked; @@ -3503,8 +3503,8 @@ TRACE_EVENT(xrep_iunlink_reload_ondisk, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->agno = XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino); - __entry->agino = XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino); + __entry->agno = XFS_INODE_TO_AGNO(ip); + __entry->agino = XFS_INODE_TO_AGINO(ip); __entry->nlink = VFS_I(ip)->i_nlink; __entry->next_agino = ip->i_next_unlinked; ), @@ -3590,8 +3590,8 @@ TRACE_EVENT(xrep_iunlink_relink_next, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->agno = XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino); - __entry->agino = XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino); + __entry->agno = XFS_INODE_TO_AGNO(ip); + __entry->agino = XFS_INODE_TO_AGINO(ip); __entry->next_agino = ip->i_next_unlinked; __entry->new_next_agino = next_agino; ), @@ -3616,8 +3616,8 @@ TRACE_EVENT(xrep_iunlink_relink_prev, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->agno = XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino); - __entry->agino = XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino); + __entry->agno = XFS_INODE_TO_AGNO(ip); + __entry->agino = XFS_INODE_TO_AGINO(ip); __entry->prev_agino = ip->i_prev_unlinked; __entry->new_prev_agino = prev_agino; ), diff --git a/fs/xfs/xfs_aops.c b/fs/xfs/xfs_aops.c index f279055fcea0..2a0c54256e93 100644 --- a/fs/xfs/xfs_aops.c +++ b/fs/xfs/xfs_aops.c @@ -139,6 +139,17 @@ xfs_end_ioend_write( */ error = blk_status_to_errno(ioend->io_bio.bi_status); if (unlikely(error)) { + /* + * Zoned writes update the in-core open zone accounting before + * I/O submission. A failed write leaves that state + * inconsistent, so shut down the filesystem instead of letting + * later writers wait forever for open zone space to become + * available. + */ + if (is_zoned) { + xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR); + goto done; + } if (ioend->io_flags & IOMAP_IOEND_SHARED) { ASSERT(!is_zoned); xfs_reflink_cancel_cow_range(ip, offset, size, true); @@ -268,7 +279,7 @@ xfs_discard_folio( xfs_alert_ratelimited(mp, "page discard on page "PTR_FMT", inode 0x%llx, pos %llu.", - folio, ip->i_ino, pos); + folio, I_INO(ip), pos); /* * The end of the punch range is always the offset of the first diff --git a/fs/xfs/xfs_attr_item.c b/fs/xfs/xfs_attr_item.c index deab14f31b38..a53ee2b0c54a 100644 --- a/fs/xfs/xfs_attr_item.c +++ b/fs/xfs/xfs_attr_item.c @@ -378,7 +378,7 @@ xfs_attr_log_item( * structure with fields from this xfs_attr_intent */ attrp = &attrip->attri_format; - attrp->alfi_ino = args->dp->i_ino; + attrp->alfi_ino = I_INO(args->dp); ASSERT(!(attr->xattri_op_flags & ~XFS_ATTRI_OP_FLAGS_TYPE_MASK)); attrp->alfi_op_flags = attr->xattri_op_flags; attrp->alfi_value_len = nv->value.iov_len; @@ -695,7 +695,7 @@ xfs_attri_recover_work( args->attr_filter = attrp->alfi_attr_filter & XFS_ATTRI_FILTER_MASK; args->op_flags = XFS_DA_OP_RECOVERY | XFS_DA_OP_OKNOENT | XFS_DA_OP_LOGGED; - args->owner = args->dp->i_ino; + args->owner = I_INO(args->dp); xfs_attr_sethash(args); switch (xfs_attr_intent_op(attr)) { diff --git a/fs/xfs/xfs_attr_list.c b/fs/xfs/xfs_attr_list.c index 114566b1ae5c..f751edb930b6 100644 --- a/fs/xfs/xfs_attr_list.c +++ b/fs/xfs/xfs_attr_list.c @@ -248,7 +248,7 @@ xfs_attr_node_list_lookup( goto out_corruptbuf; } - fa = xfs_da3_node_header_check(bp, dp->i_ino); + fa = xfs_da3_node_header_check(bp, I_INO(dp)); if (fa) goto out_corruptbuf; @@ -287,7 +287,7 @@ xfs_attr_node_list_lookup( } } - fa = xfs_attr3_leaf_header_check(bp, dp->i_ino); + fa = xfs_attr3_leaf_header_check(bp, I_INO(dp)); if (fa) { __xfs_buf_mark_corrupt(bp, fa); goto out_releasebuf; @@ -348,7 +348,7 @@ xfs_attr_node_list( case XFS_DA_NODE_MAGIC: case XFS_DA3_NODE_MAGIC: trace_xfs_attr_list_wrong_blk(context); - fa = xfs_da3_node_header_check(bp, dp->i_ino); + fa = xfs_da3_node_header_check(bp, I_INO(dp)); if (fa) { __xfs_buf_mark_corrupt(bp, fa); xfs_dirattr_mark_sick(dp, XFS_ATTR_FORK); @@ -359,7 +359,7 @@ xfs_attr_node_list( case XFS_ATTR_LEAF_MAGIC: case XFS_ATTR3_LEAF_MAGIC: leaf = bp->b_addr; - fa = xfs_attr3_leaf_header_check(bp, dp->i_ino); + fa = xfs_attr3_leaf_header_check(bp, I_INO(dp)); if (fa) { __xfs_buf_mark_corrupt(bp, fa); xfs_trans_brelse(context->tp, bp); @@ -417,7 +417,7 @@ need_lookup: break; cursor->blkno = leafhdr.forw; xfs_trans_brelse(context->tp, bp); - error = xfs_attr3_leaf_read(context->tp, dp, dp->i_ino, + error = xfs_attr3_leaf_read(context->tp, dp, I_INO(dp), cursor->blkno, &bp); if (error) return error; @@ -543,7 +543,7 @@ xfs_attr_leaf_list( context->cursor.blkno = 0; error = xfs_attr3_leaf_read(context->tp, context->dp, - context->dp->i_ino, 0, &bp); + I_INO(context->dp), 0, &bp); if (error) return error; diff --git a/fs/xfs/xfs_bmap_item.c b/fs/xfs/xfs_bmap_item.c index b237a25d6045..89f6e79a955f 100644 --- a/fs/xfs/xfs_bmap_item.c +++ b/fs/xfs/xfs_bmap_item.c @@ -245,7 +245,7 @@ xfs_bmap_update_diff_items( struct xfs_bmap_intent *ba = bi_entry(a); struct xfs_bmap_intent *bb = bi_entry(b); - return cmp_int(ba->bi_owner->i_ino, bb->bi_owner->i_ino); + return cmp_int(I_INO(ba->bi_owner), I_INO(bb->bi_owner)); } /* Log bmap updates in the intent item. */ @@ -266,7 +266,7 @@ xfs_bmap_update_log_item( next_extent = atomic_inc_return(&buip->bui_next_extent) - 1; ASSERT(next_extent < buip->bui_format.bui_nextents); map = &buip->bui_format.bui_extents[next_extent]; - map->me_owner = bi->bi_owner->i_ino; + map->me_owner = I_INO(bi->bi_owner); map->me_startblock = bi->bi_bmap.br_startblock; map->me_startoff = bi->bi_bmap.br_startoff; map->me_len = bi->bi_bmap.br_blockcount; diff --git a/fs/xfs/xfs_bmap_util.c b/fs/xfs/xfs_bmap_util.c index 0ab00615f1ad..3b9f262f8e91 100644 --- a/fs/xfs/xfs_bmap_util.c +++ b/fs/xfs/xfs_bmap_util.c @@ -1479,7 +1479,7 @@ xfs_swap_change_owner( struct xfs_trans *tp = *tpp; do { - error = xfs_bmbt_change_owner(tp, ip, XFS_DATA_FORK, ip->i_ino, + error = xfs_bmbt_change_owner(tp, ip, XFS_DATA_FORK, I_INO(ip), NULL); /* success or fatal error */ if (error != -EAGAIN) @@ -1631,7 +1631,7 @@ xfs_swap_extents( if (error) { xfs_notice(mp, "%s: inode 0x%llx format is incompatible for exchanging.", - __func__, ip->i_ino); + __func__, I_INO(ip)); goto out_trans_cancel; } diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c index 0cea458f1353..3ce12fe1c307 100644 --- a/fs/xfs/xfs_buf.c +++ b/fs/xfs/xfs_buf.c @@ -623,7 +623,7 @@ _xfs_buf_read( * type. If repair can't establish that, the buffer will be left in memory * with NULL buffer ops. */ -int +static int xfs_buf_reverify( struct xfs_buf *bp, const struct xfs_buf_ops *ops) diff --git a/fs/xfs/xfs_buf.h b/fs/xfs/xfs_buf.h index bf39d89f0f6d..b3cd1c7029f1 100644 --- a/fs/xfs/xfs_buf.h +++ b/fs/xfs/xfs_buf.h @@ -365,7 +365,6 @@ int xfs_configure_buftarg(struct xfs_buftarg *btp, unsigned int sectorsize, #define xfs_readonly_buftarg(buftarg) bdev_read_only((buftarg)->bt_bdev) -int xfs_buf_reverify(struct xfs_buf *bp, const struct xfs_buf_ops *ops); bool xfs_verify_magic(struct xfs_buf *bp, __be32 dmagic); bool xfs_verify_magic16(struct xfs_buf *bp, __be16 dmagic); diff --git a/fs/xfs/xfs_dir2_readdir.c b/fs/xfs/xfs_dir2_readdir.c index 60a80d4173f7..577f7a581043 100644 --- a/fs/xfs/xfs_dir2_readdir.c +++ b/fs/xfs/xfs_dir2_readdir.c @@ -84,7 +84,7 @@ xfs_dir2_sf_getdents( */ if (ctx->pos <= dot_offset) { ctx->pos = dot_offset & 0x7fffffff; - if (!dir_emit(ctx, ".", 1, dp->i_ino, DT_DIR)) + if (!dir_emit(ctx, ".", 1, I_INO(dp), DT_DIR)) return 0; } @@ -532,7 +532,7 @@ xfs_readdir( args.dp = dp; args.geo = dp->i_mount->m_dir_geo; args.trans = tp; - args.owner = dp->i_ino; + args.owner = I_INO(dp); if (dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) return xfs_dir2_sf_getdents(&args, ctx); diff --git a/fs/xfs/xfs_dquot.c b/fs/xfs/xfs_dquot.c index 69e9bc588c8b..c311f61d9554 100644 --- a/fs/xfs/xfs_dquot.c +++ b/fs/xfs/xfs_dquot.c @@ -1216,6 +1216,14 @@ xfs_qm_dqflush_check( type != XFS_DQTYPE_PROJ) return __this_address; + /* bigtime flag should never be set on root dquots */ + if (dqp->q_type & XFS_DQTYPE_BIGTIME) { + if (!xfs_has_bigtime(dqp->q_mount)) + return __this_address; + if (dqp->q_id == 0) + return __this_address; + } + if (dqp->q_id == 0) return NULL; @@ -1231,14 +1239,6 @@ xfs_qm_dqflush_check( !dqp->q_rtb.timer) return __this_address; - /* bigtime flag should never be set on root dquots */ - if (dqp->q_type & XFS_DQTYPE_BIGTIME) { - if (!xfs_has_bigtime(dqp->q_mount)) - return __this_address; - if (dqp->q_id == 0) - return __this_address; - } - return NULL; } diff --git a/fs/xfs/xfs_error.c b/fs/xfs/xfs_error.c index d652240a1dca..9ece70fd94ca 100644 --- a/fs/xfs/xfs_error.c +++ b/fs/xfs/xfs_error.c @@ -369,7 +369,7 @@ xfs_inode_verifier_error( xfs_alert(mp, "Metadata %s detected at %pS, inode 0x%llx %s", error == -EFSBADCRC ? "CRC error" : "corruption", - fa, ip->i_ino, name); + fa, I_INO(ip), name); xfs_alert(mp, "Unmount and run xfs_repair"); diff --git a/fs/xfs/xfs_exchmaps_item.c b/fs/xfs/xfs_exchmaps_item.c index 13a42467370f..c3745d33e54e 100644 --- a/fs/xfs/xfs_exchmaps_item.c +++ b/fs/xfs/xfs_exchmaps_item.c @@ -227,9 +227,9 @@ xfs_exchmaps_create_intent( xmi_lip = xfs_xmi_init(tp->t_mountp); xlf = &xmi_lip->xmi_format; - xlf->xmi_inode1 = xmi->xmi_ip1->i_ino; + xlf->xmi_inode1 = I_INO(xmi->xmi_ip1); xlf->xmi_igen1 = VFS_I(xmi->xmi_ip1)->i_generation; - xlf->xmi_inode2 = xmi->xmi_ip2->i_ino; + xlf->xmi_inode2 = I_INO(xmi->xmi_ip2); xlf->xmi_igen2 = VFS_I(xmi->xmi_ip2)->i_generation; xlf->xmi_startoff1 = xmi->xmi_startoff1; xlf->xmi_startoff2 = xmi->xmi_startoff2; diff --git a/fs/xfs/xfs_exchrange.c b/fs/xfs/xfs_exchrange.c index 5c083f29ea65..94965a6c2187 100644 --- a/fs/xfs/xfs_exchrange.c +++ b/fs/xfs/xfs_exchrange.c @@ -91,7 +91,7 @@ xfs_exchrange_check_freshness( trace_xfs_exchrange_freshness(fxr, ip2); /* Check that file2 hasn't otherwise been modified. */ - if (fxr->file2_ino != ip2->i_ino || + if (fxr->file2_ino != inode2->i_ino || fxr->file2_gen != inode2->i_generation || !timespec64_equal(&fxr->file2_ctime, &ctime) || !timespec64_equal(&fxr->file2_mtime, &mtime)) @@ -863,7 +863,7 @@ xfs_ioc_start_commit( kern_f->file2_ctime_nsec = kstat.ctime.tv_nsec; kern_f->file2_mtime = kstat.mtime.tv_sec; kern_f->file2_mtime_nsec = kstat.mtime.tv_nsec; - kern_f->file2_ino = ip2->i_ino; + kern_f->file2_ino = inode2->i_ino; kern_f->file2_gen = inode2->i_generation; kern_f->magic = XCR_FRESH_MAGIC; xfs_iunlock(ip2, lockflags); diff --git a/fs/xfs/xfs_export.c b/fs/xfs/xfs_export.c index 9b2ad3786b19..9afda52ed1a2 100644 --- a/fs/xfs/xfs_export.c +++ b/fs/xfs/xfs_export.c @@ -82,19 +82,19 @@ xfs_fs_encode_fh( switch (fileid_type) { case FILEID_INO32_GEN_PARENT: - fid->i32.parent_ino = XFS_I(parent)->i_ino; + fid->i32.parent_ino = parent->i_ino; fid->i32.parent_gen = parent->i_generation; fallthrough; case FILEID_INO32_GEN: - fid->i32.ino = XFS_I(inode)->i_ino; + fid->i32.ino = inode->i_ino; fid->i32.gen = inode->i_generation; break; case FILEID_INO32_GEN_PARENT | XFS_FILEID_TYPE_64FLAG: - fid64->parent_ino = XFS_I(parent)->i_ino; + fid64->parent_ino = parent->i_ino; fid64->parent_gen = parent->i_generation; fallthrough; case FILEID_INO32_GEN | XFS_FILEID_TYPE_64FLAG: - fid64->ino = XFS_I(inode)->i_ino; + fid64->ino = inode->i_ino; fid64->gen = inode->i_generation; break; } diff --git a/fs/xfs/xfs_filestream.c b/fs/xfs/xfs_filestream.c index 32e641d76e92..bb46852075f2 100644 --- a/fs/xfs/xfs_filestream.c +++ b/fs/xfs/xfs_filestream.c @@ -225,7 +225,7 @@ xfs_filestream_lookup_association( atomic_inc(&pag_group(pag)->xg_active_ref); xfs_mru_cache_done(mp->m_filestream); - trace_xfs_filestream_lookup(pag, ap->ip->i_ino); + trace_xfs_filestream_lookup(pag, I_INO(ap->ip)); ap->blkno = xfs_agbno_to_fsb(pag, 0); xfs_bmap_adjacent(ap); @@ -345,7 +345,7 @@ xfs_filestream_select_ag( args->total = ap->total; pip = xfs_filestream_get_parent(ap->ip); if (pip) { - ino = pip->i_ino; + ino = I_INO(pip); error = xfs_filestream_lookup_association(ap, args, ino, longest); xfs_irele(pip); @@ -370,7 +370,7 @@ void xfs_filestream_deassociate( struct xfs_inode *ip) { - xfs_mru_cache_delete(ip->i_mount->m_filestream, ip->i_ino); + xfs_mru_cache_delete(ip->i_mount->m_filestream, I_INO(ip)); } int diff --git a/fs/xfs/xfs_fsops.c b/fs/xfs/xfs_fsops.c index 8d64d904d73c..67624a804a7f 100644 --- a/fs/xfs/xfs_fsops.c +++ b/fs/xfs/xfs_fsops.c @@ -124,7 +124,9 @@ xfs_growfs_data_private( mp->m_sb.sb_rextsize); if (error) return error; - xfs_growfs_compute_deltas(mp, nb, &delta, &nagcount); + + nagcount = xfs_growfs_compute_agcount(mp, &nb); + delta = nb - mp->m_sb.sb_dblocks; /* * Reject filesystems with a single AG because they are not diff --git a/fs/xfs/xfs_handle.c b/fs/xfs/xfs_handle.c index 2b8617ae7ec2..0689cade8f74 100644 --- a/fs/xfs/xfs_handle.c +++ b/fs/xfs/xfs_handle.c @@ -124,7 +124,7 @@ xfs_find_handle( if (cmd == XFS_IOC_PATH_TO_FSHANDLE) hsize = xfs_fshandle_init(ip->i_mount, &handle); else - hsize = xfs_filehandle_init(ip->i_mount, ip->i_ino, + hsize = xfs_filehandle_init(ip->i_mount, inode->i_ino, inode->i_generation, &handle); error = -EFAULT; @@ -808,7 +808,7 @@ xfs_getparents( sizeof(struct xfs_attrlist_cursor)); /* Is this the root directory? */ - if (ip->i_ino == mp->m_sb.sb_rootino) + if (I_INO(ip) == mp->m_sb.sb_rootino) gp->gp_oflags |= XFS_GETPARENTS_OFLAG_ROOT; if (gpx->context.seen_enough == 0) { diff --git a/fs/xfs/xfs_healthmon.c b/fs/xfs/xfs_healthmon.c index 26c325d34bd1..4521ffdab9f1 100644 --- a/fs/xfs/xfs_healthmon.c +++ b/fs/xfs/xfs_healthmon.c @@ -533,7 +533,7 @@ xfs_healthmon_report_inode( struct xfs_healthmon_event event = { .type = type, .domain = XFS_HEALTHMON_INODE, - .ino = ip->i_ino, + .ino = I_INO(ip), .gen = VFS_I(ip)->i_generation, }; struct xfs_healthmon *hm = xfs_healthmon_get(ip->i_mount); @@ -646,7 +646,7 @@ xfs_healthmon_report_file_ioerror( struct xfs_healthmon_event event = { .type = file_ioerr_type(p->type), .domain = XFS_HEALTHMON_FILERANGE, - .fino = ip->i_ino, + .fino = I_INO(ip), .fgen = VFS_I(ip)->i_generation, .fpos = p->pos, .flen = p->len, @@ -1182,7 +1182,7 @@ xfs_ioc_health_monitor( */ if (!capable(CAP_SYS_ADMIN)) return -EPERM; - if (ip->i_ino != mp->m_sb.sb_rootino) + if (I_INO(ip) != mp->m_sb.sb_rootino) return -EPERM; if (current_user_ns() != &init_user_ns) return -EPERM; diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 2040a9292ee6..9d8dd30bd927 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -101,6 +101,7 @@ xfs_inode_alloc( return NULL; } + VFS_I(ip)->i_ino = ino; /* VFS doesn't initialise i_mode! */ VFS_I(ip)->i_mode = 0; mapping_set_folio_min_order(VFS_I(ip)->i_mapping, @@ -108,10 +109,8 @@ xfs_inode_alloc( XFS_STATS_INC(mp, xs_inodes_active); ASSERT(atomic_read(&ip->i_pincount) == 0); - ASSERT(ip->i_ino == 0); /* initialise the xfs inode */ - ip->i_ino = ino; ip->i_mount = mp; memset(&ip->i_imap, 0, sizeof(struct xfs_imap)); ip->i_cowfp = NULL; @@ -193,7 +192,7 @@ xfs_inode_free( */ spin_lock(&ip->i_flags_lock); ip->i_flags = XFS_IRECLAIM; - ip->i_ino = 0; + VFS_I(ip)->i_ino = 0; spin_unlock(&ip->i_flags_lock); __xfs_inode_free(ip); @@ -329,6 +328,7 @@ xfs_reinit_inode( struct inode *inode) { int error; + u64 ino = inode->i_ino; uint32_t nlink = inode->i_nlink; uint32_t generation = inode->i_generation; uint64_t version = inode_peek_iversion(inode); @@ -340,6 +340,7 @@ xfs_reinit_inode( error = inode_init_always(mp->m_super, inode); + inode->i_ino = ino; set_nlink(inode, nlink); inode->i_generation = generation; inode_set_iversion_queried(inode, version); @@ -397,7 +398,7 @@ xfs_iget_recycle( */ ip->i_flags &= ~XFS_IRECLAIM_RESET_FLAGS; ip->i_flags |= XFS_INEW; - xfs_perag_clear_inode_tag(pag, XFS_INO_TO_AGINO(mp, ip->i_ino), + xfs_perag_clear_inode_tag(pag, XFS_INODE_TO_AGINO(ip), XFS_ICI_RECLAIM_TAG); inode_state_assign_raw(inode, I_NEW); spin_unlock(&ip->i_flags_lock); @@ -426,9 +427,8 @@ xfs_iget_check_free_state( if (VFS_I(ip)->i_mode != 0) { xfs_warn(ip->i_mount, "Corruption detected! Free inode 0x%llx not marked free! (mode 0x%x)", - ip->i_ino, VFS_I(ip)->i_mode); - xfs_agno_mark_sick(ip->i_mount, - XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino), + I_INO(ip), VFS_I(ip)->i_mode); + xfs_agno_mark_sick(ip->i_mount, XFS_INODE_TO_AGNO(ip), XFS_SICK_AG_INOBT); return -EFSCORRUPTED; } @@ -436,9 +436,8 @@ xfs_iget_check_free_state( if (ip->i_nblocks != 0) { xfs_warn(ip->i_mount, "Corruption detected! Free inode 0x%llx has blocks allocated!", - ip->i_ino); - xfs_agno_mark_sick(ip->i_mount, - XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino), + I_INO(ip)); + xfs_agno_mark_sick(ip->i_mount, XFS_INODE_TO_AGNO(ip), XFS_SICK_AG_INOBT); return -EFSCORRUPTED; } @@ -516,7 +515,7 @@ xfs_iget_cache_hit( * will not match, so check for that, too. */ spin_lock(&ip->i_flags_lock); - if (ip->i_ino != ino) + if (I_INO(ip) != ino) goto out_skip; /* @@ -646,7 +645,7 @@ xfs_iget_cache_miss( */ xfs_iflags_set(ip, XFS_INEW); - error = xfs_imap(pag, tp, ip->i_ino, &ip->i_imap, flags); + error = xfs_imap(pag, tp, I_INO(ip), &ip->i_imap, flags); if (error) goto out_destroy; @@ -664,7 +663,7 @@ xfs_iget_cache_miss( } else { struct xfs_buf *bp; - error = xfs_imap_to_bp(mp, tp, &ip->i_imap, &bp); + error = xfs_read_icluster(pag, tp, ip->i_imap.im_agbno, &bp); if (error) goto out_destroy; @@ -819,8 +818,11 @@ again: * now. If it's a new inode being created, xfs_init_new_inode will * handle it. */ - if (xfs_iflags_test(ip, XFS_INEW) && VFS_I(ip)->i_mode != 0) - xfs_setup_existing_inode(ip); + if (xfs_iflags_test(ip, XFS_INEW) && VFS_I(ip)->i_mode != 0) { + xfs_setup_inode(ip); + xfs_setup_iops(ip); + xfs_finish_inode_setup(ip); + } return 0; out_error_or_again: @@ -962,7 +964,7 @@ xfs_reclaim_inode( struct xfs_inode *ip, struct xfs_perag *pag) { - xfs_ino_t ino = ip->i_ino; /* for radix_tree_delete */ + xfs_ino_t ino = I_INO(ip); /* for radix_tree_delete */ if (!xfs_ilock_nowait(ip, XFS_ILOCK_EXCL)) goto out; @@ -1010,7 +1012,7 @@ reclaim: */ spin_lock(&ip->i_flags_lock); ip->i_flags = XFS_IRECLAIM; - ip->i_ino = 0; + VFS_I(ip)->i_ino = 0; ip->i_sick = 0; ip->i_checked = 0; spin_unlock(&ip->i_flags_lock); @@ -1287,10 +1289,10 @@ xfs_blockgc_set_iflag( ip->i_flags |= iflag; spin_unlock(&ip->i_flags_lock); - pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino)); + pag = xfs_perag_get(mp, XFS_INODE_TO_AGNO(ip)); spin_lock(&pag->pag_ici_lock); - xfs_perag_set_inode_tag(pag, XFS_INO_TO_AGINO(mp, ip->i_ino), + xfs_perag_set_inode_tag(pag, XFS_INODE_TO_AGINO(ip), XFS_ICI_BLOCKGC_TAG); spin_unlock(&pag->pag_ici_lock); @@ -1324,10 +1326,10 @@ xfs_blockgc_clear_iflag( if (!clear_tag) return; - pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino)); + pag = xfs_perag_get(mp, XFS_INODE_TO_AGNO(ip)); spin_lock(&pag->pag_ici_lock); - xfs_perag_clear_inode_tag(pag, XFS_INO_TO_AGINO(mp, ip->i_ino), + xfs_perag_clear_inode_tag(pag, XFS_INODE_TO_AGINO(ip), XFS_ICI_BLOCKGC_TAG); spin_unlock(&pag->pag_ici_lock); @@ -1510,7 +1512,7 @@ xfs_blockgc_igrab( /* Check for stale RCU freed inode */ spin_lock(&ip->i_flags_lock); - if (!ip->i_ino) + if (!I_INO(ip)) goto out_unlock_noent; if (ip->i_flags & XFS_BLOCKGC_NOGRAB_IFLAGS) @@ -1802,10 +1804,10 @@ restart: * us to see this inode, so another lookup from the * same index will not find it again. */ - if (XFS_INO_TO_AGNO(mp, ip->i_ino) != pag_agno(pag)) + if (XFS_INODE_TO_AGNO(ip) != pag_agno(pag)) continue; - first_index = XFS_INO_TO_AGINO(mp, ip->i_ino + 1); - if (first_index < XFS_INO_TO_AGINO(mp, ip->i_ino)) + first_index = XFS_INO_TO_AGINO(mp, I_INO(ip) + 1); + if (first_index < XFS_INODE_TO_AGINO(ip)) done = true; } @@ -1892,7 +1894,7 @@ xfs_check_delalloc( if (isnullstartblock(got.br_startblock)) { xfs_warn(ip->i_mount, "ino %llx %s fork has delalloc extent at [0x%llx:0x%llx]", - ip->i_ino, + I_INO(ip), whichfork == XFS_DATA_FORK ? "data" : "cow", got.br_startoff, got.br_blockcount); } @@ -1916,14 +1918,14 @@ xfs_inodegc_set_reclaimable( ASSERT(0); } - pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino)); + pag = xfs_perag_get(mp, XFS_INODE_TO_AGNO(ip)); spin_lock(&pag->pag_ici_lock); spin_lock(&ip->i_flags_lock); trace_xfs_inode_set_reclaimable(ip); ip->i_flags &= ~(XFS_NEED_INACTIVE | XFS_INACTIVATING); ip->i_flags |= XFS_IRECLAIMABLE; - xfs_perag_set_inode_tag(pag, XFS_INO_TO_AGINO(mp, ip->i_ino), + xfs_perag_set_inode_tag(pag, XFS_INODE_TO_AGINO(ip), XFS_ICI_RECLAIM_TAG); spin_unlock(&ip->i_flags_lock); diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index ddf2707c8894..317eb57f989f 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -495,9 +495,9 @@ xfs_lock_two_inodes( ASSERT(!(ip1_mode & (XFS_IOLOCK_SHARED|XFS_IOLOCK_EXCL))); ASSERT(!(ip0_mode & (XFS_MMAPLOCK_SHARED|XFS_MMAPLOCK_EXCL))); ASSERT(!(ip1_mode & (XFS_MMAPLOCK_SHARED|XFS_MMAPLOCK_EXCL))); - ASSERT(ip0->i_ino != ip1->i_ino); + ASSERT(I_INO(ip0) != I_INO(ip1)); - if (ip0->i_ino > ip1->i_ino) { + if (I_INO(ip0) > I_INO(ip1)) { swap(ip0, ip1); swap(ip0_mode, ip1_mode); } @@ -1357,7 +1357,7 @@ xfs_inactive_health( if (sick & XFS_SICK_INO_FORGET) return; - pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino)); + pag = xfs_perag_get(mp, XFS_INODE_TO_AGNO(ip)); if (!pag) { /* There had better still be a perag structure! */ ASSERT(0); @@ -1380,7 +1380,7 @@ int xfs_inactive( xfs_inode_t *ip) { - struct xfs_mount *mp; + struct xfs_mount *mp = ip->i_mount; int error = 0; int truncate = 0; @@ -1393,7 +1393,20 @@ xfs_inactive( goto out; } - mp = ip->i_mount; + /* + * If the filesystem has been shut down - for example a mount that + * failed after background inactivation was enabled - do not + * inactivate the inode. Inactivation modifies persistent metadata, + * its transactions cannot complete on a shut down mount, and the + * subsystems it relies on (e.g. quota, mp->m_quotainfo) may not be + * set up. The attached dquots are dropped at the out: label and the + * inode then goes straight to reclaim, the same way + * xfs_inode_needs_inactive() already declines to inactivate on a shut + * down mount at queue time. + */ + if (xfs_is_shutdown(mp)) + goto out; + ASSERT(!xfs_iflags_test(ip, XFS_IRECOVERY)); xfs_inactive_health(ip); @@ -1513,7 +1526,7 @@ xfs_iunlink_lookup( * Inode in RCU freeing limbo should not happen. Warn about this and * let the caller handle the failure. */ - if (WARN_ON_ONCE(!ip->i_ino)) { + if (WARN_ON_ONCE(!I_INO(ip))) { rcu_read_unlock(); return NULL; } @@ -1614,7 +1627,7 @@ retry: * valid, the wrong inode or stale. */ spin_lock(&ip->i_flags_lock); - if (ip->i_ino != inum || __xfs_iflags_test(ip, XFS_ISTALE)) + if (I_INO(ip) != inum || __xfs_iflags_test(ip, XFS_ISTALE)) goto out_iflags_unlock; /* @@ -1796,7 +1809,7 @@ xfs_ifree( ASSERT(ip->i_disk_size == 0 || !S_ISREG(VFS_I(ip)->i_mode)); ASSERT(ip->i_nblocks == 0); - pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino)); + pag = xfs_perag_get(mp, XFS_INODE_TO_AGNO(ip)); error = xfs_inode_uninit(tp, pag, ip, &xic); if (error) @@ -2055,7 +2068,7 @@ xfs_sort_inodes( */ for (i = 0; i < num_inodes; i++) { for (j = 1; j < num_inodes; j++) { - if (i_tab[j]->i_ino < i_tab[j-1]->i_ino) + if (I_INO(i_tab[j]) < I_INO(i_tab[j-1])) swap(i_tab[j], i_tab[j - 1]); } } @@ -2305,8 +2318,7 @@ retry: struct xfs_perag *pag; struct xfs_buf *bp; - pag = xfs_perag_get(mp, - XFS_INO_TO_AGNO(mp, inodes[i]->i_ino)); + pag = xfs_perag_get(mp, XFS_INODE_TO_AGNO(inodes[i])); error = xfs_read_agi(pag, tp, 0, &bp); xfs_perag_put(pag); if (error) @@ -2387,7 +2399,7 @@ xfs_iflush( XFS_TEST_ERROR(mp, XFS_ERRTAG_IFLUSH_1)) { xfs_alert_tag(mp, XFS_PTAG_IFLUSH, "%s: Bad inode %llu magic number 0x%x, ptr "PTR_FMT, - __func__, ip->i_ino, be16_to_cpu(dip->di_magic), dip); + __func__, I_INO(ip), be16_to_cpu(dip->di_magic), dip); goto flush_out; } if (ip->i_df.if_format == XFS_DINODE_FMT_META_BTREE) { @@ -2396,7 +2408,7 @@ xfs_iflush( xfs_alert_tag(mp, XFS_PTAG_IFLUSH, "%s: Bad %s meta btree inode %Lu, ptr "PTR_FMT, __func__, xfs_metafile_type_str(ip->i_metatype), - ip->i_ino, ip); + I_INO(ip), ip); goto flush_out; } } else if (S_ISREG(VFS_I(ip)->i_mode)) { @@ -2405,7 +2417,7 @@ xfs_iflush( XFS_TEST_ERROR(mp, XFS_ERRTAG_IFLUSH_3)) { xfs_alert_tag(mp, XFS_PTAG_IFLUSH, "%s: Bad regular inode %llu, ptr "PTR_FMT, - __func__, ip->i_ino, ip); + __func__, I_INO(ip), ip); goto flush_out; } } else if (S_ISDIR(VFS_I(ip)->i_mode)) { @@ -2415,7 +2427,7 @@ xfs_iflush( XFS_TEST_ERROR(mp, XFS_ERRTAG_IFLUSH_4)) { xfs_alert_tag(mp, XFS_PTAG_IFLUSH, "%s: Bad directory inode %llu, ptr "PTR_FMT, - __func__, ip->i_ino, ip); + __func__, I_INO(ip), ip); goto flush_out; } } @@ -2424,7 +2436,7 @@ xfs_iflush( xfs_alert_tag(mp, XFS_PTAG_IFLUSH, "%s: detected corrupt incore inode %llu, " "total extents = %llu nblocks = %lld, ptr "PTR_FMT, - __func__, ip->i_ino, + __func__, I_INO(ip), ip->i_df.if_nextents + xfs_ifork_nextents(&ip->i_af), ip->i_nblocks, ip); goto flush_out; @@ -2433,7 +2445,7 @@ xfs_iflush( XFS_TEST_ERROR(mp, XFS_ERRTAG_IFLUSH_6)) { xfs_alert_tag(mp, XFS_PTAG_IFLUSH, "%s: bad inode %llu, forkoff 0x%x, ptr "PTR_FMT, - __func__, ip->i_ino, ip->i_forkoff, ip); + __func__, I_INO(ip), ip->i_forkoff, ip); goto flush_out; } @@ -2441,7 +2453,7 @@ xfs_iflush( ip->i_af.if_format == XFS_DINODE_FMT_META_BTREE) { xfs_alert_tag(mp, XFS_PTAG_IFLUSH, "%s: meta btree in inode %Lu attr fork, ptr "PTR_FMT, - __func__, ip->i_ino, ip); + __func__, I_INO(ip), ip); goto flush_out; } @@ -2741,7 +2753,7 @@ xfs_mmaplock_two_inodes_and_break_dax_layout( { int error; - if (ip1->i_ino > ip2->i_ino) + if (I_INO(ip1) > I_INO(ip2)) swap(ip1, ip2); again: @@ -2853,8 +2865,8 @@ xfs_inode_reload_unlinked_bucket( struct xfs_buf *agibp; struct xfs_agi *agi; struct xfs_perag *pag; - xfs_agnumber_t agno = XFS_INO_TO_AGNO(mp, ip->i_ino); - xfs_agino_t agino = XFS_INO_TO_AGINO(mp, ip->i_ino); + xfs_agnumber_t agno = XFS_INODE_TO_AGNO(ip); + xfs_agino_t agino = XFS_INODE_TO_AGINO(ip); xfs_agino_t prev_agino, next_agino; unsigned int bucket; bool foundit = false; diff --git a/fs/xfs/xfs_inode.h b/fs/xfs/xfs_inode.h index bd6d33557194..34c1038ebfcd 100644 --- a/fs/xfs/xfs_inode.h +++ b/fs/xfs/xfs_inode.h @@ -30,7 +30,6 @@ typedef struct xfs_inode { struct xfs_dquot *i_pdquot; /* project dquot */ /* Inode location stuff */ - xfs_ino_t i_ino; /* inode number (agno/agino)*/ struct xfs_imap i_imap; /* location for xfs_imap() */ /* Extent information. */ @@ -184,6 +183,11 @@ static inline const struct inode *VFS_IC(const struct xfs_inode *ip) return &ip->i_vnode; } +static inline xfs_ino_t I_INO(const struct xfs_inode *ip) +{ + return VFS_IC(ip)->i_ino; +} + /* * For regular files we only update the on-disk filesize when actually * writing data back to disk. Until then only the copy in the VFS inode @@ -299,9 +303,9 @@ static inline bool xfs_is_internal_inode(const struct xfs_inode *ip) * Before metadata directories, the only metadata inodes were the * three quota files, the realtime bitmap, and the realtime summary. */ - return ip->i_ino == mp->m_sb.sb_rbmino || - ip->i_ino == mp->m_sb.sb_rsumino || - xfs_is_quota_inode(&mp->m_sb, ip->i_ino); + return I_INO(ip) == mp->m_sb.sb_rbmino || + I_INO(ip) == mp->m_sb.sb_rsumino || + xfs_is_quota_inode(&mp->m_sb, I_INO(ip)); } static inline bool xfs_is_zoned_inode(const struct xfs_inode *ip) @@ -630,13 +634,6 @@ static inline void xfs_finish_inode_setup(struct xfs_inode *ip) unlock_new_inode(VFS_I(ip)); } -static inline void xfs_setup_existing_inode(struct xfs_inode *ip) -{ - xfs_setup_inode(ip); - xfs_setup_iops(ip); - xfs_finish_inode_setup(ip); -} - void xfs_irele(struct xfs_inode *ip); extern struct kmem_cache *xfs_inode_cache; diff --git a/fs/xfs/xfs_inode_item.c b/fs/xfs/xfs_inode_item.c index 4ae81eed0442..99d6ecccdaa7 100644 --- a/fs/xfs/xfs_inode_item.c +++ b/fs/xfs/xfs_inode_item.c @@ -18,6 +18,7 @@ #include "xfs_buf_item.h" #include "xfs_log.h" #include "xfs_log_priv.h" +#include "xfs_ag.h" #include "xfs_error.h" #include "xfs_rtbitmap.h" @@ -34,7 +35,7 @@ static uint64_t xfs_inode_item_sort( struct xfs_log_item *lip) { - return INODE_ITEM(lip)->ili_inode->i_ino; + return I_INO(INODE_ITEM(lip)->ili_inode); } #ifdef DEBUG_EXPENSIVE @@ -54,7 +55,7 @@ xfs_inode_item_precommit_check( xfs_inode_to_disk(ip, dip, 0); xfs_dinode_calc_crc(mp, dip); - fa = xfs_dinode_verify(mp, ip->i_ino, dip); + fa = xfs_dinode_verify(mp, I_INO(ip), dip); if (fa) { xfs_inode_verifier_error(ip, -EFSCORRUPTED, __func__, dip, sizeof(*dip), fa); @@ -104,6 +105,7 @@ xfs_inode_item_precommit( { struct xfs_inode_log_item *iip = INODE_ITEM(lip); struct xfs_inode *ip = iip->ili_inode; + struct xfs_mount *mp = ip->i_mount; struct inode *inode = VFS_I(ip); unsigned int flags = iip->ili_dirty_flags; @@ -124,8 +126,7 @@ xfs_inode_item_precommit( * to upgrade this inode to bigtime format, do so now. */ if ((flags & (XFS_ILOG_CORE | XFS_ILOG_TIMESTAMP)) && - xfs_has_bigtime(ip->i_mount) && - !xfs_inode_has_bigtime(ip)) { + xfs_has_bigtime(mp) && !xfs_inode_has_bigtime(ip)) { ip->i_diflags2 |= XFS_DIFLAG2_BIGTIME; flags |= XFS_ILOG_CORE; } @@ -138,14 +139,14 @@ xfs_inode_item_precommit( */ if (ip->i_diflags & XFS_DIFLAG_RTINHERIT) { if ((ip->i_diflags & XFS_DIFLAG_EXTSZINHERIT) && - xfs_extlen_to_rtxmod(ip->i_mount, ip->i_extsize) > 0) { + xfs_extlen_to_rtxmod(mp, ip->i_extsize) > 0) { ip->i_diflags &= ~(XFS_DIFLAG_EXTSIZE | XFS_DIFLAG_EXTSZINHERIT); ip->i_extsize = 0; flags |= XFS_ILOG_CORE; } if ((ip->i_diflags2 & XFS_DIFLAG2_COWEXTSIZE) && - xfs_extlen_to_rtxmod(ip->i_mount, ip->i_cowextsize) > 0) { + xfs_extlen_to_rtxmod(mp, ip->i_cowextsize) > 0) { ip->i_diflags2 &= ~XFS_DIFLAG2_COWEXTSIZE; ip->i_cowextsize = 0; flags |= XFS_ILOG_CORE; @@ -154,6 +155,7 @@ xfs_inode_item_precommit( spin_lock(&iip->ili_lock); if (!iip->ili_item.li_buf) { + struct xfs_perag *pag; struct xfs_buf *bp; int error; @@ -167,7 +169,9 @@ xfs_inode_item_precommit( * here. */ spin_unlock(&iip->ili_lock); - error = xfs_imap_to_bp(ip->i_mount, tp, &ip->i_imap, &bp); + pag = xfs_perag_get(mp, XFS_INODE_TO_AGNO(ip)); + error = xfs_read_icluster(pag, tp, ip->i_imap.im_agbno, &bp); + xfs_perag_put(pag); if (error) return error; @@ -588,7 +592,7 @@ xfs_inode_to_log_dinode( to->di_flags2 = ip->i_diflags2; /* also covers the di_used_blocks union arm: */ to->di_cowextsize = ip->i_cowextsize; - to->di_ino = ip->i_ino; + to->di_ino = I_INO(ip); to->di_lsn = lsn; memset(to->di_pad2, 0, sizeof(to->di_pad2)); uuid_copy(&to->di_uuid, &ip->i_mount->m_sb.sb_meta_uuid); @@ -647,13 +651,15 @@ xfs_inode_item_format( { struct xfs_inode_log_item *iip = INODE_ITEM(lip); struct xfs_inode *ip = iip->ili_inode; + struct xfs_mount *mp = ip->i_mount; struct xfs_inode_log_format *ilf; ilf = xlog_format_start(lfb, XLOG_REG_TYPE_IFORMAT); ilf->ilf_type = XFS_LI_INODE; - ilf->ilf_ino = ip->i_ino; - ilf->ilf_blkno = ip->i_imap.im_blkno; - ilf->ilf_len = ip->i_imap.im_len; + ilf->ilf_ino = I_INO(ip); + ilf->ilf_blkno = XFS_AGB_TO_DADDR(mp, XFS_INODE_TO_AGNO(ip), + ip->i_imap.im_agbno); + ilf->ilf_len = XFS_FSB_TO_BB(mp, M_IGEO(mp)->blocks_per_cluster); ilf->ilf_boffset = ip->i_imap.im_boffset; ilf->ilf_fields = XFS_ILOG_CORE; ilf->ilf_size = 2; /* format + core */ diff --git a/fs/xfs/xfs_inode_item_recover.c b/fs/xfs/xfs_inode_item_recover.c index b42106c4a9ef..169a8fe3bf0a 100644 --- a/fs/xfs/xfs_inode_item_recover.c +++ b/fs/xfs/xfs_inode_item_recover.c @@ -99,7 +99,7 @@ xfs_recover_inode_owner_change( if (in_f->ilf_fields & XFS_ILOG_DOWNER) { ASSERT(in_f->ilf_fields & XFS_ILOG_DBROOT); error = xfs_bmbt_change_owner(NULL, ip, XFS_DATA_FORK, - ip->i_ino, buffer_list); + I_INO(ip), buffer_list); if (error) goto out_free_ip; } @@ -107,7 +107,7 @@ xfs_recover_inode_owner_change( if (in_f->ilf_fields & XFS_ILOG_AOWNER) { ASSERT(in_f->ilf_fields & XFS_ILOG_ABROOT); error = xfs_bmbt_change_owner(NULL, ip, XFS_ATTR_FORK, - ip->i_ino, buffer_list); + I_INO(ip), buffer_list); if (error) goto out_free_ip; } diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c index 904fc3d32449..1b53701bebea 100644 --- a/fs/xfs/xfs_ioctl.c +++ b/fs/xfs/xfs_ioctl.c @@ -1005,7 +1005,7 @@ xfs_ioc_swapext( if (ip->i_mount != tip->i_mount) return -EINVAL; - if (ip->i_ino == tip->i_ino) + if (I_INO(ip) == I_INO(tip)) return -EINVAL; if (xfs_is_shutdown(ip->i_mount)) diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index f20a02f49ed9..225c3de88d03 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -45,7 +45,7 @@ xfs_alert_fsblock_zero( "Access to block zero in inode %llu " "start_block: %llx start_off: %llx " "blkcnt: %llx extent-state: %x", - (unsigned long long)ip->i_ino, + (unsigned long long)I_INO(ip), (unsigned long long)imap->br_startblock, (unsigned long long)imap->br_startoff, (unsigned long long)imap->br_blockcount, @@ -113,6 +113,7 @@ xfs_bmbt_to_iomap( return xfs_alert_fsblock_zero(ip, imap); } + iomap->flags = iomap_flags; if (imap->br_startblock == HOLESTARTBLOCK) { iomap->addr = IOMAP_NULL_ADDR; iomap->type = IOMAP_HOLE; @@ -143,7 +144,6 @@ xfs_bmbt_to_iomap( } iomap->offset = XFS_FSB_TO_B(mp, imap->br_startoff); iomap->length = XFS_FSB_TO_B(mp, imap->br_blockcount); - iomap->flags = iomap_flags; if (mapping_flags & IOMAP_DAX) { iomap->dax_dev = target->bt_daxdev; } else { @@ -1736,7 +1736,7 @@ restart: if (count_fsb > ac->reserved_blocks) { xfs_warn_ratelimited(mp, "Short write on ino 0x%llx comm %.20s due to three-way race with write fault and direct I/O", - ip->i_ino, current->comm); + I_INO(ip), current->comm); count_fsb = ac->reserved_blocks; if (!count_fsb) { error = -EIO; diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c index 325c2200c501..6339f4956ecb 100644 --- a/fs/xfs/xfs_iops.c +++ b/fs/xfs/xfs_iops.c @@ -703,7 +703,7 @@ xfs_vn_getattr( stat->nlink = inode->i_nlink; stat->uid = vfsuid_into_kuid(vfsuid); stat->gid = vfsgid_into_kgid(vfsgid); - stat->ino = ip->i_ino; + stat->ino = inode->i_ino; stat->atime = inode_get_atime(inode); fill_mg_cmtime(stat, request_mask, inode); @@ -1423,7 +1423,6 @@ xfs_setup_inode( gfp_t gfp_mask; bool is_meta = xfs_is_internal_inode(ip); - inode->i_ino = ip->i_ino; inode_state_set_raw(inode, I_NEW); inode_sb_list_add(inode); diff --git a/fs/xfs/xfs_itable.c b/fs/xfs/xfs_itable.c index 97f106d2b4cd..159295c63e8f 100644 --- a/fs/xfs/xfs_itable.c +++ b/fs/xfs/xfs_itable.c @@ -97,7 +97,7 @@ xfs_bulkstat_one_int( } ASSERT(ip != NULL); - ASSERT(ip->i_imap.im_blkno != 0); + ASSERT(ip->i_imap.im_agbno != 0); inode = VFS_I(ip); vfsuid = i_uid_into_vfsuid(idmap, inode); vfsgid = i_gid_into_vfsgid(idmap, inode); diff --git a/fs/xfs/xfs_iunlink_item.c b/fs/xfs/xfs_iunlink_item.c index a03a48eeb9a8..d76d3f0434f1 100644 --- a/fs/xfs/xfs_iunlink_item.c +++ b/fs/xfs/xfs_iunlink_item.c @@ -40,7 +40,7 @@ static uint64_t xfs_iunlink_item_sort( struct xfs_log_item *lip) { - return IUL_ITEM(lip)->ip->i_ino; + return I_INO(IUL_ITEM(lip)->ip); } /* @@ -59,7 +59,7 @@ xfs_iunlink_log_dinode( int offset; int error; - error = xfs_imap_to_bp(tp->t_mountp, tp, &ip->i_imap, &ibp); + error = xfs_read_icluster(iup->pag, tp, ip->i_imap.im_agbno, &ibp); if (error) return error; /* diff --git a/fs/xfs/xfs_log.h b/fs/xfs/xfs_log.h index 0f23812b0b31..ca66429bf6c9 100644 --- a/fs/xfs/xfs_log.h +++ b/fs/xfs/xfs_log.h @@ -80,23 +80,6 @@ xlog_format_copy( return buf; } -/* - * By comparing each component, we don't have to worry about extra - * endian issues in treating two 32 bit numbers as one 64 bit number - */ -static inline xfs_lsn_t _lsn_cmp(xfs_lsn_t lsn1, xfs_lsn_t lsn2) -{ - if (CYCLE_LSN(lsn1) != CYCLE_LSN(lsn2)) - return (CYCLE_LSN(lsn1)= 0 && feat < XFS_EXPERIMENTAL_MAX); BUILD_BUG_ON(ARRAY_SIZE(features) != XFS_EXPERIMENTAL_MAX); diff --git a/fs/xfs/xfs_message.h b/fs/xfs/xfs_message.h index 49b0ef40d299..811b885f41c3 100644 --- a/fs/xfs/xfs_message.h +++ b/fs/xfs/xfs_message.h @@ -93,7 +93,6 @@ void xfs_buf_alert_ratelimited(struct xfs_buf *bp, const char *rlmsg, enum xfs_experimental_feat { XFS_EXPERIMENTAL_SHRINK, XFS_EXPERIMENTAL_LARP, - XFS_EXPERIMENTAL_ZONED, XFS_EXPERIMENTAL_MAX, }; diff --git a/fs/xfs/xfs_mount.c b/fs/xfs/xfs_mount.c index 7aa51826b1ca..be90c7b03994 100644 --- a/fs/xfs/xfs_mount.c +++ b/fs/xfs/xfs_mount.c @@ -1080,7 +1080,7 @@ xfs_mountfs( if (XFS_IS_CORRUPT(mp, !S_ISDIR(VFS_I(rip)->i_mode))) { xfs_warn(mp, "corrupted root inode %llu: not a directory", - (unsigned long long)rip->i_ino); + (unsigned long long)I_INO(rip)); xfs_iunlock(rip, XFS_ILOCK_EXCL); error = -EFSCORRUPTED; goto out_rele_rip; @@ -1246,11 +1246,19 @@ xfs_mountfs( xfs_irele(mp->m_metadirip); /* - * Inactivate all inodes that might still be in memory after a log - * intent recovery failure so that reclaim can free them. Metadata - * inodes and the root directory shouldn't need inactivation, but the - * mount failed for some reason, so pull down all the state and flee. + * The mount has failed. Mark the filesystem shut down so that any + * inodes still queued for background inactivation are dropped + * straight to reclaim instead of being inactivated: a failed mount + * must not write to the (possibly corrupt, only partially set up) + * persistent metadata, and parts of the mount it would need - e.g. + * the quota subsystem (mp->m_quotainfo) - may never have been + * initialised. + * + * Flush the queue so that those inodes are pulled down and reclaim + * can free them; with the fs shut down xfs_inodegc_inactivate() + * turns each one reclaimable without touching the on-disk structures. */ + xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR); xfs_inodegc_flush(mp); /* diff --git a/fs/xfs/xfs_mount.h b/fs/xfs/xfs_mount.h index d964bae096ef..66a02d1b9ad7 100644 --- a/fs/xfs/xfs_mount.h +++ b/fs/xfs/xfs_mount.h @@ -580,8 +580,6 @@ __XFS_HAS_FEAT(nouuid, NOUUID) #define XFS_OPSTATE_WARNED_METADIR 17 /* Filesystem should use qflags to determine quotaon status */ #define XFS_OPSTATE_RESUMING_QUOTAON 18 -/* Kernel has logged a warning about zoned RT device being used on this fs. */ -#define XFS_OPSTATE_WARNED_ZONED 19 /* (Zoned) GC is in progress */ #define XFS_OPSTATE_ZONEGC_RUNNING 20 diff --git a/fs/xfs/xfs_qm.c b/fs/xfs/xfs_qm.c index 7bd15d9e7fba..aa0d2976f1c3 100644 --- a/fs/xfs/xfs_qm.c +++ b/fs/xfs/xfs_qm.c @@ -297,7 +297,7 @@ xfs_qm_need_dqattach( return false; if (!XFS_NOT_DQATTACHED(mp, ip)) return false; - if (xfs_is_quota_inode(&mp->m_sb, ip->i_ino)) + if (xfs_is_quota_inode(&mp->m_sb, I_INO(ip))) return false; if (xfs_is_metadir_inode(ip)) return false; @@ -390,7 +390,7 @@ xfs_qm_dqdetach( trace_xfs_dquot_dqdetach(ip); - ASSERT(!xfs_is_quota_inode(&ip->i_mount->m_sb, ip->i_ino)); + ASSERT(!xfs_is_quota_inode(&ip->i_mount->m_sb, I_INO(ip))); if (ip->i_udquot) { xfs_qm_dqrele(ip->i_udquot); ip->i_udquot = NULL; @@ -986,11 +986,11 @@ xfs_qm_qino_alloc( mp->m_sb.sb_qflags = mp->m_qflags & XFS_ALL_QUOTA_ACCT; } if (flags & XFS_QMOPT_UQUOTA) - mp->m_sb.sb_uquotino = (*ipp)->i_ino; + mp->m_sb.sb_uquotino = I_INO(*ipp); else if (flags & XFS_QMOPT_GQUOTA) - mp->m_sb.sb_gquotino = (*ipp)->i_ino; + mp->m_sb.sb_gquotino = I_INO(*ipp); else - mp->m_sb.sb_pquotino = (*ipp)->i_ino; + mp->m_sb.sb_pquotino = I_INO(*ipp); spin_unlock(&mp->m_sb_lock); xfs_log_sb(tp); diff --git a/fs/xfs/xfs_quotaops.c b/fs/xfs/xfs_quotaops.c index 8804508cc2b8..81f739e3aa0c 100644 --- a/fs/xfs/xfs_quotaops.c +++ b/fs/xfs/xfs_quotaops.c @@ -34,7 +34,7 @@ xfs_qm_fill_state( defq = xfs_get_defquota(mp->m_quotainfo, type); - tstate->ino = ip->i_ino; + tstate->ino = I_INO(ip); tstate->flags |= QCI_SYSFILE; tstate->blocks = ip->i_nblocks; tstate->nextents = ip->i_df.if_nextents; diff --git a/fs/xfs/xfs_reflink.c b/fs/xfs/xfs_reflink.c index 251dec48f0e3..a5c188b78138 100644 --- a/fs/xfs/xfs_reflink.c +++ b/fs/xfs/xfs_reflink.c @@ -1188,7 +1188,7 @@ xfs_reflink_set_inode_flag( goto out_error; /* Lock both files against IO */ - if (src->i_ino == dest->i_ino) + if (I_INO(src) == I_INO(dest)) xfs_ilock(src, XFS_ILOCK_EXCL); else xfs_lock_two_inodes(src, XFS_ILOCK_EXCL, dest, XFS_ILOCK_EXCL); @@ -1202,7 +1202,7 @@ xfs_reflink_set_inode_flag( } else xfs_iunlock(src, XFS_ILOCK_EXCL); - if (src->i_ino == dest->i_ino) + if (I_INO(src) == I_INO(dest)) goto commit_flags; if (!xfs_is_reflink_inode(dest)) { diff --git a/fs/xfs/xfs_rtalloc.c b/fs/xfs/xfs_rtalloc.c index 153f3c378f9f..7a3f97686989 100644 --- a/fs/xfs/xfs_rtalloc.c +++ b/fs/xfs/xfs_rtalloc.c @@ -890,8 +890,7 @@ xfs_growfs_rt_sb_fields( static int xfs_growfs_rt_zoned( - struct xfs_rtgroup *rtg, - xfs_rfsblock_t nrblocks) + struct xfs_rtgroup *rtg) { struct xfs_mount *mp = rtg_mount(rtg); struct xfs_mount *nmp; @@ -903,7 +902,8 @@ xfs_growfs_rt_zoned( * Calculate new sb and mount fields for this round. Also ensure the * rtg_extents value is uptodate as the rtbitmap code relies on it. */ - nmp = xfs_growfs_rt_alloc_fake_mount(mp, nrblocks, + nmp = xfs_growfs_rt_alloc_fake_mount(mp, + xfs_rtgs_to_rfsbs(mp, rtg_rgno(rtg) + 1), mp->m_sb.sb_rextsize); if (!nmp) return -ENOMEM; @@ -933,6 +933,14 @@ xfs_growfs_rt_zoned( mp->m_features |= XFS_FEAT_REALTIME; xfs_rtrmapbt_compute_maxlevels(mp); xfs_rtrefcountbt_compute_maxlevels(mp); + + /* + * Finally add the newly added zone to the freelist and add the space + * to the available counter. The order is important here: only add + * the available space after the zones, as available space guarantees + * that zones to back it are available. + */ + xfs_zone_mark_free(rtg); xfs_zoned_add_available(mp, freed_rtx); out_free: kfree(nmp); @@ -1218,7 +1226,7 @@ xfs_growfs_rtg( } if (xfs_has_zoned(mp)) { - error = xfs_growfs_rt_zoned(rtg, nrblocks); + error = xfs_growfs_rt_zoned(rtg); goto out_rele; } diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index f8de44443e81..eac7f9503805 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -1901,7 +1901,6 @@ xfs_fs_fill_super( error = -EINVAL; goto out_filestream_unmount; } - xfs_warn_experimental(mp, XFS_EXPERIMENTAL_ZONED); } if (xfs_has_reflink(mp)) { diff --git a/fs/xfs/xfs_symlink.c b/fs/xfs/xfs_symlink.c index c4da624fb296..5585ac7f4d16 100644 --- a/fs/xfs/xfs_symlink.c +++ b/fs/xfs/xfs_symlink.c @@ -52,7 +52,7 @@ xfs_readlink( if (pathlen < 0 || pathlen > XFS_SYMLINK_MAXLEN) { xfs_alert(mp, "%s: inode (%llu) bad symlink length (%lld)", - __func__, (unsigned long long) ip->i_ino, + __func__, (unsigned long long)I_INO(ip), (long long) pathlen); ASSERT(0); goto out_corrupt; @@ -186,7 +186,7 @@ xfs_symlink( xfs_qm_vop_create_dqattach(tp, du.ip, udqp, gdqp, pdqp); resblks -= XFS_IALLOC_SPACE_RES(mp); - error = xfs_symlink_write_target(tp, du.ip, du.ip->i_ino, target_path, + error = xfs_symlink_write_target(tp, du.ip, I_INO(du.ip), target_path, pathlen, fs_blocks, resblks); if (error) goto out_trans_cancel; @@ -339,7 +339,7 @@ xfs_inactive_symlink( if (pathlen <= 0 || pathlen > XFS_SYMLINK_MAXLEN) { xfs_alert(mp, "%s: inode (0x%llx) bad symlink length (%d)", - __func__, (unsigned long long)ip->i_ino, pathlen); + __func__, (unsigned long long)I_INO(ip), pathlen); xfs_iunlock(ip, XFS_ILOCK_EXCL); ASSERT(0); xfs_inode_mark_sick(ip, XFS_SICK_INO_SYMLINK); diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h index f87c738d84b2..d478693674f9 100644 --- a/fs/xfs/xfs_trace.h +++ b/fs/xfs/xfs_trace.h @@ -132,7 +132,7 @@ DECLARE_EVENT_CLASS(xfs_attr_list_class, ), TP_fast_assign( __entry->dev = VFS_I(ctx->dp)->i_sb->s_dev; - __entry->ino = ctx->dp->i_ino; + __entry->ino = I_INO(ctx->dp); __entry->hashval = ctx->cursor.hashval; __entry->blkno = ctx->cursor.blkno; __entry->offset = ctx->cursor.offset; @@ -634,7 +634,7 @@ TRACE_EVENT(xfs_attr_list_node_descend, ), TP_fast_assign( __entry->dev = VFS_I(ctx->dp)->i_sb->s_dev; - __entry->ino = ctx->dp->i_ino; + __entry->ino = I_INO(ctx->dp); __entry->hashval = ctx->cursor.hashval; __entry->blkno = ctx->cursor.blkno; __entry->offset = ctx->cursor.offset; @@ -688,7 +688,7 @@ DECLARE_EVENT_CLASS(xfs_bmap_class, ifp = xfs_iext_state_to_fork(ip, state); xfs_iext_get_extent(ifp, cur, &r); __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->leaf = cur->leaf; __entry->pos = cur->pos; __entry->startoff = r.br_startoff; @@ -1021,7 +1021,7 @@ DECLARE_EVENT_CLASS(xfs_lock_class, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->lock_flags = lock_flags; __entry->caller_ip = caller_ip; ), @@ -1052,7 +1052,7 @@ DECLARE_EVENT_CLASS(xfs_inode_class, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->iflags = ip->i_flags; ), TP_printk("dev %d:%d ino 0x%llx iflags 0x%lx", @@ -1128,7 +1128,7 @@ DECLARE_EVENT_CLASS(xfs_fault_class, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->order = order; ), TP_printk("dev %d:%d ino 0x%llx order %u", @@ -1157,7 +1157,7 @@ DECLARE_EVENT_CLASS(xfs_iref_class, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->count = icount_read_once(VFS_I(ip)); __entry->pincount = atomic_read(&ip->i_pincount); __entry->iflags = ip->i_flags; @@ -1185,7 +1185,7 @@ TRACE_EVENT(xfs_iomap_prealloc_size, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->blocks = blocks; __entry->shift = shift; __entry->writeio_blocks = writeio_blocks; @@ -1272,7 +1272,7 @@ DECLARE_EVENT_CLASS(xfs_namespace_class, ), TP_fast_assign( __entry->dev = VFS_I(dp)->i_sb->s_dev; - __entry->dp_ino = dp->i_ino; + __entry->dp_ino = I_INO(dp); __entry->namelen = name->len; memcpy(__get_str(name), name->name, name->len); ), @@ -1308,8 +1308,8 @@ TRACE_EVENT(xfs_rename, ), TP_fast_assign( __entry->dev = VFS_I(src_dp)->i_sb->s_dev; - __entry->src_dp_ino = src_dp->i_ino; - __entry->target_dp_ino = target_dp->i_ino; + __entry->src_dp_ino = I_INO(src_dp); + __entry->target_dp_ino = I_INO(target_dp); __entry->src_namelen = src_name->len; __entry->target_namelen = target_name->len; memcpy(__get_str(src_name), src_name->name, src_name->len); @@ -1760,7 +1760,7 @@ DECLARE_EVENT_CLASS(xfs_file_class, ), TP_fast_assign( __entry->dev = file_inode(iocb->ki_filp)->i_sb->s_dev; - __entry->ino = XFS_I(file_inode(iocb->ki_filp))->i_ino; + __entry->ino = file_inode(iocb->ki_filp)->i_ino; __entry->size = XFS_I(file_inode(iocb->ki_filp))->i_disk_size; __entry->offset = iocb->ki_pos; __entry->count = iov_iter_count(iter); @@ -1796,7 +1796,7 @@ TRACE_EVENT(xfs_iomap_atomic_write_cow, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->offset = offset; __entry->count = count; ), @@ -1824,7 +1824,7 @@ DECLARE_EVENT_CLASS(xfs_imap_class, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->size = ip->i_disk_size; __entry->offset = offset; __entry->count = count; @@ -1869,7 +1869,7 @@ DECLARE_EVENT_CLASS(xfs_simple_io_class, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->isize = VFS_I(ip)->i_size; __entry->disize = ip->i_disk_size; __entry->offset = offset; @@ -1908,7 +1908,7 @@ DECLARE_EVENT_CLASS(xfs_itrunc_class, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->size = ip->i_disk_size; __entry->new_size = new_size; ), @@ -1941,7 +1941,7 @@ TRACE_EVENT(xfs_bunmap, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->size = ip->i_disk_size; __entry->fileoff = fileoff; __entry->len = len; @@ -2344,7 +2344,7 @@ DECLARE_EVENT_CLASS(xfs_da_class, ), TP_fast_assign( __entry->dev = VFS_I(args->dp)->i_sb->s_dev; - __entry->ino = args->dp->i_ino; + __entry->ino = I_INO(args->dp); if (args->namelen) memcpy(__get_str(name), args->name, args->namelen); __entry->namelen = args->namelen; @@ -2411,7 +2411,7 @@ DECLARE_EVENT_CLASS(xfs_attr_class, ), TP_fast_assign( __entry->dev = VFS_I(args->dp)->i_sb->s_dev; - __entry->ino = args->dp->i_ino; + __entry->ino = I_INO(args->dp); if (args->namelen) memcpy(__get_str(name), args->name, args->namelen); __entry->namelen = args->namelen; @@ -2511,7 +2511,7 @@ DECLARE_EVENT_CLASS(xfs_dir2_space_class, ), TP_fast_assign( __entry->dev = VFS_I(args->dp)->i_sb->s_dev; - __entry->ino = args->dp->i_ino; + __entry->ino = I_INO(args->dp); __entry->op_flags = args->op_flags; __entry->idx = idx; ), @@ -2544,7 +2544,7 @@ TRACE_EVENT(xfs_dir2_leafn_moveents, ), TP_fast_assign( __entry->dev = VFS_I(args->dp)->i_sb->s_dev; - __entry->ino = args->dp->i_ino; + __entry->ino = I_INO(args->dp); __entry->op_flags = args->op_flags; __entry->src_idx = src_idx; __entry->dst_idx = dst_idx; @@ -2586,7 +2586,7 @@ DECLARE_EVENT_CLASS(xfs_swap_extent_class, TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; __entry->which = which; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->format = ip->i_df.if_format; __entry->nex = ip->i_df.if_nextents; __entry->broot_size = ip->i_df.if_broot_bytes; @@ -2944,7 +2944,7 @@ TRACE_EVENT(xfs_btree_alloc_block, switch (cur->bc_ops->type) { case XFS_BTREE_TYPE_INODE: __entry->agno = 0; - __entry->ino = cur->bc_ino.ip->i_ino; + __entry->ino = I_INO(cur->bc_ino.ip); break; case XFS_BTREE_TYPE_AG: __entry->agno = cur->bc_group->xg_gno; @@ -2996,7 +2996,7 @@ TRACE_EVENT(xfs_btree_free_block, __entry->agno = xfs_daddr_to_agno(cur->bc_mp, xfs_buf_daddr(bp)); if (cur->bc_ops->type == XFS_BTREE_TYPE_INODE) - __entry->ino = cur->bc_ino.ip->i_ino; + __entry->ino = I_INO(cur->bc_ino.ip); else __entry->ino = 0; __assign_str(name); @@ -3251,7 +3251,7 @@ DECLARE_EVENT_CLASS(xfs_btree_error_class, switch (cur->bc_ops->type) { case XFS_BTREE_TYPE_INODE: __entry->agno = 0; - __entry->ino = cur->bc_ino.ip->i_ino; + __entry->ino = I_INO(cur->bc_ino.ip); break; case XFS_BTREE_TYPE_AG: __entry->agno = cur->bc_group->xg_gno; @@ -3470,7 +3470,7 @@ DECLARE_EVENT_CLASS(xfs_bmap_deferred_class, bi->bi_bmap.br_startblock, bi->bi_group->xg_type); } - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->whichfork = bi->bi_whichfork; __entry->l_loff = bi->bi_bmap.br_startoff; __entry->l_len = bi->bi_bmap.br_blockcount; @@ -3974,7 +3974,7 @@ DECLARE_EVENT_CLASS(xfs_inode_error_class, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->error = error; __entry->caller_ip = caller_ip; ), @@ -4012,12 +4012,12 @@ DECLARE_EVENT_CLASS(xfs_double_io_class, ), TP_fast_assign( __entry->dev = VFS_I(src)->i_sb->s_dev; - __entry->src_ino = src->i_ino; + __entry->src_ino = I_INO(src); __entry->src_isize = VFS_I(src)->i_size; __entry->src_disize = src->i_disk_size; __entry->src_offset = soffset; __entry->len = len; - __entry->dest_ino = dest->i_ino; + __entry->dest_ino = I_INO(dest); __entry->dest_isize = VFS_I(dest)->i_size; __entry->dest_disize = dest->i_disk_size; __entry->dest_offset = doffset; @@ -4057,7 +4057,7 @@ DECLARE_EVENT_CLASS(xfs_inode_irec_class, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->lblk = irec->br_startoff; __entry->len = irec->br_blockcount; __entry->pblk = irec->br_startblock; @@ -4093,7 +4093,7 @@ DECLARE_EVENT_CLASS(xfs_wb_invalid_class, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->addr = iomap->addr; __entry->pos = iomap->offset; __entry->len = iomap->length; @@ -4136,7 +4136,7 @@ DECLARE_EVENT_CLASS(xfs_iomap_invalid_class, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->addr = iomap->addr; __entry->pos = iomap->offset; __entry->len = iomap->length; @@ -4183,10 +4183,10 @@ TRACE_EVENT(xfs_reflink_remap_blocks, ), TP_fast_assign( __entry->dev = VFS_I(src)->i_sb->s_dev; - __entry->src_ino = src->i_ino; + __entry->src_ino = I_INO(src); __entry->src_lblk = soffset; __entry->len = len; - __entry->dest_ino = dest->i_ino; + __entry->dest_ino = I_INO(dest); __entry->dest_lblk = doffset; ), TP_printk("dev %d:%d fsbcount 0x%llx " @@ -4504,8 +4504,7 @@ TRACE_EVENT(xfs_iunlink_update_dinode, TP_fast_assign( __entry->dev = pag_mount(iup->pag)->m_super->s_dev; __entry->agno = pag_agno(iup->pag); - __entry->agino = - XFS_INO_TO_AGINO(iup->ip->i_mount, iup->ip->i_ino); + __entry->agino = XFS_INODE_TO_AGINO(iup->ip); __entry->old_ptr = old_ptr; __entry->new_ptr = iup->next_agino; ), @@ -4529,8 +4528,8 @@ TRACE_EVENT(xfs_iunlink_reload_next, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->agno = XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino); - __entry->agino = XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino); + __entry->agno = XFS_INODE_TO_AGNO(ip); + __entry->agino = XFS_INODE_TO_AGINO(ip); __entry->prev_agino = ip->i_prev_unlinked; __entry->next_agino = ip->i_next_unlinked; ), @@ -4552,8 +4551,8 @@ TRACE_EVENT(xfs_inode_reload_unlinked_bucket, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->agno = XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino); - __entry->agino = XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino); + __entry->agno = XFS_INODE_TO_AGNO(ip); + __entry->agino = XFS_INODE_TO_AGINO(ip); ), TP_printk("dev %d:%d agno 0x%x agino 0x%x bucket %u", MAJOR(__entry->dev), MINOR(__entry->dev), @@ -4572,8 +4571,8 @@ DECLARE_EVENT_CLASS(xfs_ag_inode_class, ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->agno = XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino); - __entry->agino = XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino); + __entry->agno = XFS_INODE_TO_AGNO(ip); + __entry->agino = XFS_INODE_TO_AGINO(ip); ), TP_printk("dev %d:%d agno 0x%x agino 0x%x", MAJOR(__entry->dev), MINOR(__entry->dev), @@ -4650,7 +4649,7 @@ DECLARE_EVENT_CLASS(xfs_inode_corrupt_class, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->flags = flags; ), TP_printk("dev %d:%d ino 0x%llx flags 0x%x", @@ -4769,10 +4768,8 @@ TRACE_EVENT(xfs_btree_commit_ifakeroot, TP_fast_assign( __entry->dev = cur->bc_mp->m_super->s_dev; __assign_str(name); - __entry->agno = XFS_INO_TO_AGNO(cur->bc_mp, - cur->bc_ino.ip->i_ino); - __entry->agino = XFS_INO_TO_AGINO(cur->bc_mp, - cur->bc_ino.ip->i_ino); + __entry->agno = XFS_INODE_TO_AGNO(cur->bc_ino.ip); + __entry->agino = XFS_INODE_TO_AGINO(cur->bc_ino.ip); __entry->levels = cur->bc_ino.ifake->if_levels; __entry->blocks = cur->bc_ino.ifake->if_blocks; __entry->whichfork = cur->bc_ino.whichfork; @@ -5031,7 +5028,7 @@ DECLARE_EVENT_CLASS(xfs_das_state_class, ), TP_fast_assign( __entry->das = das; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); ), TP_printk("state change %s ino 0x%llx", __print_symbolic(__entry->das, XFS_DAS_STRINGS), @@ -5296,7 +5293,7 @@ DECLARE_EVENT_CLASS(xfs_exchrange_inode_class, TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; __entry->whichfile = whichfile; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->format = ip->i_df.if_format; __entry->nex = ip->i_df.if_nextents; __entry->fork_off = xfs_inode_fork_boff(ip); @@ -5349,10 +5346,10 @@ DECLARE_EVENT_CLASS(xfs_exchrange_class, ), TP_fast_assign( __entry->dev = VFS_I(ip1)->i_sb->s_dev; - __entry->ip1_ino = ip1->i_ino; + __entry->ip1_ino = I_INO(ip1); __entry->ip1_isize = VFS_I(ip1)->i_size; __entry->ip1_disize = ip1->i_disk_size; - __entry->ip2_ino = ip2->i_ino; + __entry->ip2_ino = I_INO(ip2); __entry->ip2_isize = VFS_I(ip2)->i_size; __entry->ip2_disize = ip2->i_disk_size; @@ -5408,7 +5405,7 @@ TRACE_EVENT(xfs_exchrange_freshness, struct inode *inode2 = VFS_I(ip2); __entry->dev = inode2->i_sb->s_dev; - __entry->ip2_ino = ip2->i_ino; + __entry->ip2_ino = I_INO(ip2); ts64 = inode_get_ctime(inode2); __entry->ip2_ctime = ts64.tv_sec; @@ -5480,8 +5477,8 @@ DECLARE_EVENT_CLASS(xfs_exchmaps_estimate_class, ), TP_fast_assign( __entry->dev = req->ip1->i_mount->m_super->s_dev; - __entry->ino1 = req->ip1->i_ino; - __entry->ino2 = req->ip2->i_ino; + __entry->ino1 = I_INO(req->ip1); + __entry->ino2 = I_INO(req->ip2); __entry->startoff1 = req->startoff1; __entry->startoff2 = req->startoff2; __entry->blockcount = req->blockcount; @@ -5532,8 +5529,8 @@ DECLARE_EVENT_CLASS(xfs_exchmaps_intent_class, ), TP_fast_assign( __entry->dev = mp->m_super->s_dev; - __entry->ino1 = xmi->xmi_ip1->i_ino; - __entry->ino2 = xmi->xmi_ip2->i_ino; + __entry->ino1 = I_INO(xmi->xmi_ip1); + __entry->ino2 = I_INO(xmi->xmi_ip2); __entry->flags = xmi->xmi_flags; __entry->startoff1 = xmi->xmi_startoff1; __entry->startoff2 = xmi->xmi_startoff2; @@ -5628,8 +5625,8 @@ TRACE_EVENT(xfs_exchmaps_delta_nextents, int whichfork = xfs_exchmaps_reqfork(req); __entry->dev = req->ip1->i_mount->m_super->s_dev; - __entry->ino1 = req->ip1->i_ino; - __entry->ino2 = req->ip2->i_ino; + __entry->ino1 = I_INO(req->ip1); + __entry->ino2 = I_INO(req->ip2); __entry->nexts1 = xfs_ifork_ptr(req->ip1, whichfork)->if_nextents; __entry->nexts2 = xfs_ifork_ptr(req->ip2, whichfork)->if_nextents; __entry->d_nexts1 = d_nexts1; @@ -5659,7 +5656,7 @@ DECLARE_EVENT_CLASS(xfs_getparents_rec_class, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->firstu = context->firstu; __entry->reclen = pptr->gpr_reclen; __entry->bufsize = ppi->gp_bufsize; @@ -5703,7 +5700,7 @@ DECLARE_EVENT_CLASS(xfs_getparents_class, ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; - __entry->ino = ip->i_ino; + __entry->ino = I_INO(ip); __entry->iflags = ppi->gp_iflags; __entry->oflags = ppi->gp_oflags; __entry->bufsize = ppi->gp_bufsize; @@ -5742,8 +5739,8 @@ DECLARE_EVENT_CLASS(xfs_metadir_update_class, ), TP_fast_assign( __entry->dev = upd->dp->i_mount->m_super->s_dev; - __entry->dp_ino = upd->dp->i_ino; - __entry->ino = upd->ip ? upd->ip->i_ino : NULLFSINO; + __entry->dp_ino = I_INO(upd->dp); + __entry->ino = upd->ip ? I_INO(upd->ip) : NULLFSINO; __assign_str(fname); ), TP_printk("dev %d:%d dp 0x%llx fname '%s' ino 0x%llx", @@ -5777,8 +5774,8 @@ DECLARE_EVENT_CLASS(xfs_metadir_update_error_class, ), TP_fast_assign( __entry->dev = upd->dp->i_mount->m_super->s_dev; - __entry->dp_ino = upd->dp->i_ino; - __entry->ino = upd->ip ? upd->ip->i_ino : NULLFSINO; + __entry->dp_ino = I_INO(upd->dp); + __entry->ino = upd->ip ? I_INO(upd->ip) : NULLFSINO; __entry->error = error; __assign_str(fname); ), @@ -5810,7 +5807,7 @@ DECLARE_EVENT_CLASS(xfs_metadir_class, ), TP_fast_assign( __entry->dev = VFS_I(dp)->i_sb->s_dev; - __entry->dp_ino = dp->i_ino; + __entry->dp_ino = I_INO(dp); __entry->ino = ino, __entry->ftype = name->type; __entry->namelen = name->len; @@ -6328,7 +6325,7 @@ TRACE_EVENT(xfs_healthmon_report_file_ioerror, TP_fast_assign( __entry->dev = hm->dev; __entry->type = p->type; - __entry->ino = XFS_I(p->inode)->i_ino; + __entry->ino = p->inode->i_ino; __entry->gen = p->inode->i_generation; __entry->pos = p->pos; __entry->len = p->len; diff --git a/fs/xfs/xfs_trans.c b/fs/xfs/xfs_trans.c index 148cc32449c1..7bfbd9f6f0df 100644 --- a/fs/xfs/xfs_trans.c +++ b/fs/xfs/xfs_trans.c @@ -1164,7 +1164,7 @@ xfs_trans_reserve_more_inode( if (error) return error; - if (!XFS_IS_QUOTA_ON(mp) || xfs_is_quota_inode(&mp->m_sb, ip->i_ino)) + if (!XFS_IS_QUOTA_ON(mp) || xfs_is_quota_inode(&mp->m_sb, I_INO(ip))) return 0; if (tp->t_flags & XFS_TRANS_RESERVE) diff --git a/fs/xfs/xfs_trans_buf.c b/fs/xfs/xfs_trans_buf.c index 95db73a37e57..7e17b93fe9ad 100644 --- a/fs/xfs/xfs_trans_buf.c +++ b/fs/xfs/xfs_trans_buf.c @@ -234,7 +234,6 @@ xfs_trans_read_buf_map( const struct xfs_buf_ops *ops) { struct xfs_buf *bp = NULL; - struct xfs_buf_log_item *bip; int error; *bpp = NULL; @@ -251,9 +250,10 @@ xfs_trans_read_buf_map( if (bp) { ASSERT(xfs_buf_islocked(bp)); ASSERT(bp->b_transp == tp); - ASSERT(bp->b_log_item != NULL); ASSERT(!bp->b_error); ASSERT(bp->b_flags & XBF_DONE); + ASSERT(atomic_read(&bp->b_log_item->bli_refcount) > 0); + ASSERT(bp->b_ops); /* * We never locked this buf ourselves, so we shouldn't @@ -264,39 +264,8 @@ xfs_trans_read_buf_map( return -EIO; } - /* - * Check if the caller is trying to read a buffer that is - * already attached to the transaction yet has no buffer ops - * assigned. Ops are usually attached when the buffer is - * attached to the transaction, or by the read caller if - * special circumstances. That didn't happen, which is not - * how this is supposed to go. - * - * If the buffer passes verification we'll let this go, but if - * not we have to shut down. Let the transaction cleanup code - * release this buffer when it kills the tranaction. - */ - ASSERT(bp->b_ops != NULL); - error = xfs_buf_reverify(bp, ops); - if (error) { - xfs_buf_ioerror_alert(bp, __return_address); - - if (tp->t_flags & XFS_TRANS_DIRTY) - xfs_force_shutdown(tp->t_mountp, - SHUTDOWN_META_IO_ERROR); - - /* bad CRC means corrupted metadata */ - if (error == -EFSBADCRC) - error = -EFSCORRUPTED; - return error; - } - - bip = bp->b_log_item; - bip->bli_recur++; - - ASSERT(atomic_read(&bip->bli_refcount) > 0); - trace_xfs_trans_read_buf_recur(bip); - ASSERT(bp->b_ops != NULL || ops == NULL); + bp->b_log_item->bli_recur++; + trace_xfs_trans_read_buf_recur(bp->b_log_item); *bpp = bp; return 0; } diff --git a/fs/xfs/xfs_trans_dquot.c b/fs/xfs/xfs_trans_dquot.c index eaf9de6e07fd..1606c614f205 100644 --- a/fs/xfs/xfs_trans_dquot.c +++ b/fs/xfs/xfs_trans_dquot.c @@ -164,7 +164,7 @@ xfs_trans_mod_ino_dquot( if (xfs_hooks_switched_on(&xfs_dqtrx_hooks_switch)) { struct xfs_mod_ino_dqtrx_params p = { .tx_id = (uintptr_t)tp, - .ino = ip->i_ino, + .ino = I_INO(ip), .q_type = xfs_dquot_type(dqp), .q_id = dqp->q_id, .delta = delta @@ -247,7 +247,7 @@ xfs_trans_mod_dquot_byino( xfs_mount_t *mp = tp->t_mountp; if (!XFS_IS_QUOTA_ON(mp) || - xfs_is_quota_inode(&mp->m_sb, ip->i_ino) || + xfs_is_quota_inode(&mp->m_sb, I_INO(ip)) || xfs_is_metadir_inode(ip)) return; @@ -990,7 +990,7 @@ xfs_trans_reserve_quota_nblks( if (xfs_is_metadir_inode(ip)) return 0; - ASSERT(!xfs_is_quota_inode(&mp->m_sb, ip->i_ino)); + ASSERT(!xfs_is_quota_inode(&mp->m_sb, I_INO(ip))); xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); if (force) diff --git a/fs/xfs/xfs_xattr.c b/fs/xfs/xfs_xattr.c index 544213067d59..1efe6c8139b2 100644 --- a/fs/xfs/xfs_xattr.c +++ b/fs/xfs/xfs_xattr.c @@ -104,7 +104,7 @@ xfs_attr_change( args->op_flags |= XFS_DA_OP_LOGGED; } - args->owner = args->dp->i_ino; + args->owner = I_INO(args->dp); args->geo = mp->m_attr_geo; args->whichfork = XFS_ATTR_FORK; xfs_attr_sethash(args); diff --git a/fs/xfs/xfs_zone_alloc.c b/fs/xfs/xfs_zone_alloc.c index 5e297b75a85f..08d8b34f467e 100644 --- a/fs/xfs/xfs_zone_alloc.c +++ b/fs/xfs/xfs_zone_alloc.c @@ -944,6 +944,14 @@ xfs_zone_rgbno_is_valid( rtg_rgno(rtg), XFS_RTG_FREE); } +void +xfs_zone_mark_free( + struct xfs_rtgroup *rtg) +{ + xfs_group_set_mark(rtg_group(rtg), XFS_RTG_FREE); + atomic_inc(&rtg_mount(rtg)->m_zone_info->zi_nr_free_zones); +} + static void xfs_free_open_zones( struct xfs_zone_info *zi) @@ -1082,8 +1090,7 @@ xfs_init_zone( if (write_pointer == 0) { /* zone is empty */ - atomic_inc(&zi->zi_nr_free_zones); - xfs_group_set_mark(rtg_group(rtg), XFS_RTG_FREE); + xfs_zone_mark_free(rtg); iz->available += rtg_blocks(rtg); } else if (write_pointer < rtg_blocks(rtg)) { /* zone is open */ diff --git a/fs/xfs/xfs_zone_alloc.h b/fs/xfs/xfs_zone_alloc.h index 8b2ef98c81ef..abf8358bb266 100644 --- a/fs/xfs/xfs_zone_alloc.h +++ b/fs/xfs/xfs_zone_alloc.h @@ -42,6 +42,7 @@ void xfs_zoned_wake_all(struct xfs_mount *mp); bool xfs_zone_rgbno_is_valid(struct xfs_rtgroup *rtg, xfs_rgnumber_t rgbno); void xfs_mark_rtg_boundary(struct iomap_ioend *ioend); +void xfs_zone_mark_free(struct xfs_rtgroup *rtg); uint64_t xfs_zoned_default_resblks(struct xfs_mount *mp, enum xfs_free_counter ctr); void xfs_zoned_show_stats(struct seq_file *m, struct xfs_mount *mp); diff --git a/fs/xfs/xfs_zone_gc.c b/fs/xfs/xfs_zone_gc.c index f03211e4354a..f76a09130852 100644 --- a/fs/xfs/xfs_zone_gc.c +++ b/fs/xfs/xfs_zone_gc.c @@ -924,9 +924,7 @@ xfs_zone_gc_finish_reset( goto out; } - xfs_group_set_mark(rtg_group(rtg), XFS_RTG_FREE); - atomic_inc(&zi->zi_nr_free_zones); - + xfs_zone_mark_free(rtg); xfs_zoned_add_available(mp, rtg_blocks(rtg)); wake_up_all(&zi->zi_zone_wait); diff --git a/include/linux/hfs_common.h b/include/linux/hfs_common.h index 07dfc39630ab..45fb4c9ff9f5 100644 --- a/include/linux/hfs_common.h +++ b/include/linux/hfs_common.h @@ -513,6 +513,7 @@ struct hfs_btree_header_rec { /* HFS+ BTree misc info */ #define HFSPLUS_TREE_HEAD 0 #define HFSPLUS_NODE_MXSZ 32768 +#define HFSPLUS_NODE_MINSZ 512 #define HFSPLUS_ATTR_TREE_NODE_SIZE 8192 #define HFSPLUS_BTREE_HDR_NODE_RECS_COUNT 3 #define HFSPLUS_BTREE_HDR_MAP_REC_INDEX 2 /* Map (bitmap) record in Header node */ diff --git a/include/linux/pidfs.h b/include/linux/pidfs.h index 416bdff4d6ce..0abf7da9ab23 100644 --- a/include/linux/pidfs.h +++ b/include/linux/pidfs.h @@ -2,6 +2,8 @@ #ifndef _LINUX_PID_FS_H #define _LINUX_PID_FS_H +#include + struct coredump_params; struct file *pidfs_alloc_file(struct pid *pid, unsigned int flags); @@ -14,7 +16,21 @@ void pidfs_exit(struct task_struct *tsk); void pidfs_coredump(const struct coredump_params *cprm); #endif extern const struct dentry_operations pidfs_dentry_operations; -int pidfs_register_pid(struct pid *pid); +int pidfs_register_pid_gfp(struct pid *pid, gfp_t gfp); + +/** + * pidfs_register_pid - register a struct pid in pidfs + * @pid: pid to pin + * + * Register a struct pid in pidfs. + * + * Return: On success zero, on error a negative error code is returned. + */ +static inline int pidfs_register_pid(struct pid *pid) +{ + return pidfs_register_pid_gfp(pid, GFP_KERNEL); +} + void pidfs_free_pid(struct pid *pid); #endif /* _LINUX_PID_FS_H */ diff --git a/include/linux/stddef.h b/include/linux/stddef.h index 80b6bfb944f0..e1851c50c89b 100644 --- a/include/linux/stddef.h +++ b/include/linux/stddef.h @@ -100,6 +100,71 @@ enum { * Creates a union between a flexible-array member (FAM) in a struct and a set * of additional members that would otherwise follow it. * + * Beware that, as this helper encloses TYPE NAME and MEMBERS in the same + * union, designated initializers for MEMBERS may overwrite portions + * previously initialized through NAME. + * + * For example:: + * + * struct flex { + * size_t count; + * u8 fam[]; + * }; + * + * struct composite { + * ... + * __TRAILING_OVERLAP(struct flex, flex, fam, __packed, + * u8 data; + * ); + * } __packed; + * + * static struct composite comp = { + * .flex = { + * .count = 1, + * }, + * .data = 2, + * }; + * + * In the example above, .flex and .data initialize different views of the same + * union storage. Since .data is initialized last, it _may_ overwrite portions + * previously initialized through .flex, leading to .flex.count being zeroed + * out. + * + * A couple of alternatives are shown below. + * + * a) Initialize only one view of the overlapped storage and assign the rest + * at runtime:: + * + * static struct composite comp = { + * .flex = { + * .count = 1, + * }, + * }; + * + * static void foo(void) + * { + * comp.data = 2; + * ... + * } + * + * b) Alternatively, replace designated initializers with runtime assignments:: + * + * static void foo(void) + * { + * struct composite comp; + * + * comp.flex.count = 1; + * comp.data = 2; + * ... + * } + * + * Compiler Explorer test code: https://godbolt.org/z/voM4E36dT + * + * For another example of the above see commit 5e54510a9389 ("acpi: nfit: + * intel: avoid multiple -Wflex-array-member-not-at-end warnings") + * + * Link: https://git.kernel.org/linus/5e54510a9389caa9 + * * @TYPE: Flexible structure type name, including "struct" keyword. * @NAME: Name for a variable to define. * @FAM: The flexible-array member within @TYPE diff --git a/include/trace/events/btrfs.h b/include/trace/events/btrfs.h index ec1df8b94517..4c5c47c5edb7 100644 --- a/include/trace/events/btrfs.h +++ b/include/trace/events/btrfs.h @@ -31,6 +31,19 @@ struct btrfs_space_info; struct btrfs_raid_bio; struct raid56_bio_trace_info; struct find_free_extent_ctl; +struct btrfs_trans_handle; +struct btrfs_transaction; +struct btrfs_log_ctx; + +#define show_inode_type(mode) \ + __print_symbolic((mode) & S_IFMT, \ + { S_IFDIR, "DIR" }, \ + { S_IFREG, "REG" }, \ + { S_IFLNK, "LNK" }, \ + { S_IFIFO, "FIFO" }, \ + { S_IFCHR, "CHR" }, \ + { S_IFBLK, "BLK" }, \ + { S_IFSOCK, "SOCK" }) #define show_ref_type(type) \ __print_symbolic(type, \ @@ -101,8 +114,21 @@ struct find_free_extent_ctl; EM( ALLOC_CHUNK_FORCE, "ALLOC_CHUNK_FORCE") \ EM( RUN_DELAYED_IPUTS, "RUN_DELAYED_IPUTS") \ EM( COMMIT_TRANS, "COMMIT_TRANS") \ + EM( RECLAIM_ZONES, "RECLAIM_ZONES") \ EMe(RESET_ZONES, "RESET_ZONES") +#define TRANSACTION_STATES \ + EM( TRANS_STATE_RUNNING, "TRANS_STATE_RUNNING") \ + EM( TRANS_STATE_COMMIT_PREP, "TRANS_STATE_COMMIT_PREP") \ + EM( TRANS_STATE_COMMIT_START, "TRANS_STATE_COMMIT_START") \ + EM( TRANS_STATE_COMMIT_DOING, "TRANS_STATE_COMMIT_DOING") \ + EM( TRANS_STATE_UNBLOCKED, "TRANS_STATE_UNBLOCKED") \ + EM( TRANS_STATE_SUPER_COMMITTED, "TRANS_STATE_SUPER_COMMITTED") \ + EMe(TRANS_STATE_COMPLETED, "TRANS_STATE_COMPLETED") + +#define LOG_MODES \ + EM( LOG_INODE_ALL, "LOG_INODE_ALL") \ + EMe(LOG_INODE_EXISTS, "LOG_INODE_EXISTS") /* * First define the enums in the above macros to be exported to userspace via * TRACE_DEFINE_ENUM(). @@ -118,6 +144,8 @@ FI_TYPES QGROUP_RSV_TYPES IO_TREE_OWNER FLUSH_STATES +TRANSACTION_STATES +LOG_MODES /* * Now redefine the EM and EMe macros to map the enums to the strings that will @@ -180,25 +208,66 @@ FLUSH_STATES #define TP_printk_btrfs(fmt, args...) \ TP_printk("%pU: " fmt, __entry->fsid, args) -TRACE_EVENT(btrfs_transaction_commit, +TRACE_EVENT(btrfs_transaction_start, - TP_PROTO(const struct btrfs_fs_info *fs_info), + TP_PROTO(const struct btrfs_transaction *trans), - TP_ARGS(fs_info), + TP_ARGS(trans), TP_STRUCT__entry_btrfs( __field( u64, generation ) - __field( u64, root_objectid ) ), - TP_fast_assign_btrfs(fs_info, - __entry->generation = fs_info->generation; - __entry->root_objectid = BTRFS_ROOT_TREE_OBJECTID; + TP_fast_assign_btrfs(trans->fs_info, + __entry->generation = trans->transid; ), - TP_printk_btrfs("root=%llu(%s) gen=%llu", - show_root_type(__entry->root_objectid), - __entry->generation) + TP_printk_btrfs("gen=%llu", __entry->generation) +); + +TRACE_EVENT(btrfs_transaction_commit, + + TP_PROTO(const struct btrfs_trans_handle *trans), + + TP_ARGS(trans), + + TP_STRUCT__entry_btrfs( + __field( u64, generation ) + __field( bool, in_fsync ) + __field( int, state ) + ), + + TP_fast_assign_btrfs(trans->fs_info, + __entry->generation = trans->transid; + __entry->in_fsync = trans->in_fsync; + __entry->state = trans->transaction->state; + ), + + TP_printk_btrfs("gen=%llu in_fsync=%d state=%d(%s)", __entry->generation, + __entry->in_fsync, __entry->state, + __print_symbolic(__entry->state, TRANSACTION_STATES)) +); + +TRACE_EVENT(btrfs_transaction_abort, + + TP_PROTO(const struct btrfs_trans_handle *trans), + + TP_ARGS(trans), + + TP_STRUCT__entry_btrfs( + __field( u64, generation ) + __field( bool, in_fsync ) + __field( int, error ) + ), + + TP_fast_assign_btrfs(trans->fs_info, + __entry->generation = trans->transid; + __entry->in_fsync = trans->in_fsync; + __entry->error = trans->aborted; + ), + + TP_printk_btrfs("gen=%llu in_fsync=%d error=%d", __entry->generation, + __entry->in_fsync, __entry->error) ); DECLARE_EVENT_CLASS(btrfs__inode, @@ -670,7 +739,7 @@ TRACE_EVENT(btrfs_finish_ordered_extent, TP_printk_btrfs("root=%llu(%s) ino=%llu start=%llu len=%llu uptodate=%d", show_root_type(__entry->root_objectid), __entry->ino, __entry->start, - __entry->len, !!__entry->uptodate) + __entry->len, __entry->uptodate) ); DECLARE_EVENT_CLASS(btrfs__writepage, @@ -755,17 +824,18 @@ TRACE_EVENT(btrfs_writepage_end_io_hook, __entry->end, __entry->uptodate) ); -TRACE_EVENT(btrfs_sync_file, +TRACE_EVENT(btrfs_sync_file_enter, TP_PROTO(const struct file *file, int datasync), TP_ARGS(file, datasync), TP_STRUCT__entry_btrfs( - __field( u64, ino ) - __field( u64, parent ) - __field( int, datasync ) - __field( u64, root_objectid ) + __field( u64, ino ) + __field( umode_t, mode ) + __field( u64, parent ) + __field( int, datasync ) + __field( u64, root_objectid ) ), TP_fast_assign( @@ -778,13 +848,790 @@ TRACE_EVENT(btrfs_sync_file, __entry->parent = btrfs_ino(BTRFS_I(parent_inode)); __entry->datasync = datasync; __entry->root_objectid = btrfs_root_id(BTRFS_I(inode)->root); + __entry->mode = inode->i_mode; ), - TP_printk_btrfs("root=%llu(%s) ino=%llu parent=%llu datasync=%d", - show_root_type(__entry->root_objectid), - __entry->ino, - __entry->parent, - __entry->datasync) + TP_printk_btrfs("root=%llu(%s) ino=%llu type=%s parent=%llu datasync=%d", + show_root_type(__entry->root_objectid), __entry->ino, + show_inode_type(__entry->mode), __entry->parent, + __entry->datasync) +); + +TRACE_EVENT(btrfs_sync_file_exit, + + TP_PROTO(const struct file *file, int ret), + + TP_ARGS(file, ret), + + TP_STRUCT__entry_btrfs( + __field( u64, ino ) + __field( int, ret ) + __field( u64, root_objectid ) + ), + + TP_fast_assign( + struct btrfs_inode *inode = BTRFS_I(file_inode(file)); + + TP_fast_assign_fsid(inode->root->fs_info); + __entry->root_objectid = btrfs_root_id(inode->root); + __entry->ino = btrfs_ino(inode); + __entry->ret = ret; + ), + + TP_printk_btrfs("root=%llu(%s) ino=%llu ret=%d", + show_root_type(__entry->root_objectid), + __entry->ino, __entry->ret) +); + +TRACE_EVENT(btrfs_log_inode_parent_enter, + + TP_PROTO(const struct btrfs_trans_handle *trans, struct btrfs_inode *inode), + + TP_ARGS(trans, inode), + + TP_STRUCT__entry_btrfs( + __field( u64, root_objectid ) + __field( u64, ino ) + __field( umode_t, mode ) + __field( u64, transid ) + __field( u64, generation ) + __field( u64, logged_trans ) + __field( u64, last_unlink_trans ) + __field( int, last_sub_trans ) + __field( int, inode_last_log_commit ) + __field( int, root_last_log_commit ) + ), + + TP_fast_assign( + struct btrfs_root *root = inode->root; + + TP_fast_assign_fsid(root->fs_info); + __entry->root_objectid = btrfs_root_id(root); + __entry->ino = btrfs_ino(inode); + __entry->mode = inode->vfs_inode.i_mode; + __entry->transid = trans->transid; + __entry->generation = inode->generation; + spin_lock(&inode->lock); + __entry->logged_trans = inode->logged_trans; + __entry->last_unlink_trans = inode->last_unlink_trans; + __entry->last_sub_trans = inode->last_sub_trans; + __entry->inode_last_log_commit = inode->last_log_commit; + spin_unlock(&inode->lock); + __entry->root_last_log_commit = btrfs_get_root_last_log_commit(root); + ), + + TP_printk_btrfs("root=%llu(%s) ino=%llu type=%s transid=%llu gen=%llu" + " logged_trans=%llu last_unlink_trans=%llu last_sub_trans=%d" + " inode_last_log_commit=%d root_last_log_commit=%d", + show_root_type(__entry->root_objectid), __entry->ino, + show_inode_type(__entry->mode), __entry->transid, + __entry->generation, __entry->logged_trans, + __entry->last_unlink_trans, __entry->last_sub_trans, + __entry->inode_last_log_commit, __entry->root_last_log_commit) +); + +TRACE_EVENT(btrfs_log_inode_parent_exit, + + TP_PROTO(const struct btrfs_trans_handle *trans, + const struct btrfs_inode *inode, + int ret), + + TP_ARGS(trans, inode, ret), + + TP_STRUCT__entry_btrfs( + __field( u64, root_objectid ) + __field( u64, ino ) + __field( u64, transid ) + __field( int, ret ) + ), + + TP_fast_assign( + TP_fast_assign_fsid(inode->root->fs_info); + __entry->root_objectid = btrfs_root_id(inode->root); + __entry->ino = btrfs_ino(inode); + __entry->transid = trans->transid; + __entry->ret = ret; + ), + + TP_printk_btrfs("root=%llu(%s) ino=%llu transid=%llu ret=%d", + show_root_type(__entry->root_objectid), __entry->ino, + __entry->transid, __entry->ret) +); + +TRACE_EVENT(btrfs_log_inode_enter, + + TP_PROTO(const struct btrfs_trans_handle *trans, struct btrfs_inode *inode, + const struct btrfs_log_ctx *ctx, int log_mode), + + TP_ARGS(trans, inode, ctx, log_mode), + + TP_STRUCT__entry_btrfs( + __field( u64, root_objectid ) + __field( u64, ino ) + __field( umode_t, mode ) + __field( u64, transid ) + __field( u64, generation ) + __field( u64, logged_trans ) + __field( u64, last_unlink_trans ) + __field( u64, last_reflink_trans ) + __field( int, last_sub_trans ) + __field( int, last_log_commit ) + __field( bool, logging_new_name ) + __field( bool, logging_new_delayed_dentries ) + __field( bool, is_conflict_inode ) + __field( bool, full_sync ) + __field( bool, copy_everything ) + __field( bool, no_xattrs ) + __field( int, log_mode ) + ), + + TP_fast_assign( + TP_fast_assign_fsid(inode->root->fs_info); + __entry->root_objectid = btrfs_root_id(inode->root); + __entry->ino = btrfs_ino(inode); + __entry->mode = inode->vfs_inode.i_mode; + __entry->transid = trans->transid; + __entry->generation = inode->generation; + spin_lock(&inode->lock); + __entry->logged_trans = inode->logged_trans; + __entry->last_unlink_trans = inode->last_unlink_trans; + __entry->last_reflink_trans = inode->last_reflink_trans; + __entry->last_sub_trans = inode->last_sub_trans; + __entry->last_log_commit = inode->last_log_commit; + spin_unlock(&inode->lock); + __entry->logging_new_name = ctx->logging_new_name; + __entry->logging_new_delayed_dentries = ctx->logging_new_delayed_dentries; + __entry->is_conflict_inode = ctx->logging_conflict_inodes; + __entry->full_sync = + test_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags); + __entry->copy_everything = + test_bit(BTRFS_INODE_COPY_EVERYTHING, &inode->runtime_flags); + __entry->no_xattrs = + test_bit(BTRFS_INODE_NO_XATTRS, &inode->runtime_flags); + __entry->log_mode = log_mode; + ), + + TP_printk_btrfs("root=%llu(%s) ino=%llu type=%s transid=%llu gen=%llu" + " logged_trans=%llu last_unlink_trans=%llu" + " last_reflink_trans=%llu last_sub_trans=%d last_log_commit=%d" + " logging_new_name=%d logging_new_delayed_dentries=%d" + " is_conflict_inode=%d full_sync=%d copy_everything=%d" + " no_xattrs=%d log_mode=%d(%s)", + show_root_type(__entry->root_objectid), __entry->ino, + show_inode_type(__entry->mode), __entry->transid, + __entry->generation, __entry->logged_trans, + __entry->last_unlink_trans, __entry->last_reflink_trans, + __entry->last_sub_trans, __entry->last_log_commit, + __entry->logging_new_name, __entry->logging_new_delayed_dentries, + __entry->is_conflict_inode, __entry->log_mode, + __entry->full_sync, __entry->copy_everything, __entry->no_xattrs, + __print_symbolic(__entry->log_mode, LOG_MODES)) +); + +TRACE_EVENT(btrfs_log_inode_exit, + + TP_PROTO(const struct btrfs_trans_handle *trans, struct btrfs_inode *inode, + int ret), + + TP_ARGS(trans, inode, ret), + + TP_STRUCT__entry_btrfs( + __field( u64, root_objectid ) + __field( u64, ino ) + __field( u64, transid ) + __field( u64, logged_trans ) + __field( u64, last_reflink_trans ) + __field( int, last_sub_trans ) + __field( int, last_log_commit ) + __field( int, ret ) + ), + + TP_fast_assign( + TP_fast_assign_fsid(inode->root->fs_info); + __entry->root_objectid = btrfs_root_id(inode->root); + __entry->ino = btrfs_ino(inode); + __entry->transid = trans->transid; + spin_lock(&inode->lock); + __entry->logged_trans = inode->logged_trans; + __entry->last_reflink_trans = inode->last_reflink_trans; + __entry->last_sub_trans = inode->last_sub_trans; + __entry->last_log_commit = inode->last_log_commit; + spin_unlock(&inode->lock); + __entry->ret = ret; + ), + + TP_printk_btrfs("root=%llu(%s) ino=%llu transid=%llu logged_trans=%llu" + " last_reflink_trans=%llu last_sub_trans=%d" + " last_log_commit=%d ret=%d", + show_root_type(__entry->root_objectid), __entry->ino, + __entry->transid, __entry->logged_trans, + __entry->last_reflink_trans, __entry->last_sub_trans, + __entry->last_log_commit, __entry->ret) +); + +TRACE_EVENT(btrfs_log_all_parents_enter, + + TP_PROTO(const struct btrfs_trans_handle *trans, + const struct btrfs_inode *inode), + + TP_ARGS(trans, inode), + + TP_STRUCT__entry_btrfs( + __field( u64, root_objectid ) + __field( u64, ino ) + __field( u64, transid ) + ), + + TP_fast_assign( + TP_fast_assign_fsid(inode->root->fs_info); + __entry->root_objectid = btrfs_root_id(inode->root); + __entry->ino = btrfs_ino(inode); + __entry->transid = trans->transid; + ), + + TP_printk_btrfs("root=%llu(%s) ino=%llu transid=%llu", + show_root_type(__entry->root_objectid), __entry->ino, + __entry->transid) +); + +TRACE_EVENT(btrfs_log_all_parents_exit, + + TP_PROTO(const struct btrfs_trans_handle *trans, + const struct btrfs_inode *inode, + int ret), + + TP_ARGS(trans, inode, ret), + + TP_STRUCT__entry_btrfs( + __field( u64, root_objectid ) + __field( u64, ino ) + __field( u64, transid ) + __field( int, ret ) + ), + + TP_fast_assign( + TP_fast_assign_fsid(inode->root->fs_info); + __entry->root_objectid = btrfs_root_id(inode->root); + __entry->ino = btrfs_ino(inode); + __entry->transid = trans->transid; + __entry->ret = ret; + ), + + TP_printk_btrfs("root=%llu(%s) ino=%llu transid=%llu ret=%d", + show_root_type(__entry->root_objectid), __entry->ino, + __entry->transid, __entry->ret) +); + +TRACE_EVENT(btrfs_log_all_new_ancestors_enter, + + TP_PROTO(const struct btrfs_trans_handle *trans, + const struct btrfs_inode *inode), + + TP_ARGS(trans, inode), + + TP_STRUCT__entry_btrfs( + __field( u64, root_objectid ) + __field( u64, ino ) + __field( u64, transid ) + __field( unsigned int, nlink ) + ), + + TP_fast_assign( + TP_fast_assign_fsid(inode->root->fs_info); + __entry->root_objectid = btrfs_root_id(inode->root); + __entry->ino = btrfs_ino(inode); + __entry->transid = trans->transid; + __entry->nlink = inode->vfs_inode.i_nlink; + ), + + TP_printk_btrfs("root=%llu(%s) ino=%llu transid=%llu nlink=%u", + show_root_type(__entry->root_objectid), __entry->ino, + __entry->transid, __entry->nlink) +); + +TRACE_EVENT(btrfs_log_all_new_ancestors_exit, + + TP_PROTO(const struct btrfs_trans_handle *trans, + const struct btrfs_inode *inode, + int ret), + + TP_ARGS(trans, inode, ret), + + TP_STRUCT__entry_btrfs( + __field( u64, root_objectid ) + __field( u64, ino ) + __field( u64, transid ) + __field( int, ret ) + ), + + TP_fast_assign( + TP_fast_assign_fsid(inode->root->fs_info); + __entry->root_objectid = btrfs_root_id(inode->root); + __entry->ino = btrfs_ino(inode); + __entry->transid = trans->transid; + __entry->ret = ret; + ), + + TP_printk_btrfs("root=%llu(%s) ino=%llu transid=%llu ret=%d", + show_root_type(__entry->root_objectid), __entry->ino, + __entry->transid, __entry->ret) +); + +TRACE_EVENT(btrfs_log_new_dir_dentries_enter, + + TP_PROTO(const struct btrfs_trans_handle *trans, + const struct btrfs_inode *inode), + + TP_ARGS(trans, inode), + + TP_STRUCT__entry_btrfs( + __field( u64, root_objectid ) + __field( u64, ino ) + __field( u64, transid ) + ), + + TP_fast_assign( + TP_fast_assign_fsid(inode->root->fs_info); + __entry->root_objectid = btrfs_root_id(inode->root); + __entry->ino = btrfs_ino(inode); + __entry->transid = trans->transid; + ), + + TP_printk_btrfs("root=%llu(%s) ino=%llu transid=%llu", + show_root_type(__entry->root_objectid), __entry->ino, + __entry->transid) +); + +TRACE_EVENT(btrfs_log_new_dir_dentries_exit, + + TP_PROTO(const struct btrfs_trans_handle *trans, + const struct btrfs_inode *inode, + int ret), + + TP_ARGS(trans, inode, ret), + + TP_STRUCT__entry_btrfs( + __field( u64, root_objectid ) + __field( u64, ino ) + __field( u64, transid ) + __field( int, ret ) + ), + + TP_fast_assign( + TP_fast_assign_fsid(inode->root->fs_info); + __entry->root_objectid = btrfs_root_id(inode->root); + __entry->ino = btrfs_ino(inode); + __entry->transid = trans->transid; + __entry->ret = ret; + ), + + TP_printk_btrfs("root=%llu(%s) ino=%llu transid=%llu ret=%d", + show_root_type(__entry->root_objectid), __entry->ino, + __entry->transid, __entry->ret) +); + +TRACE_EVENT(btrfs_add_conflicting_inode_enter, + + TP_PROTO(const struct btrfs_trans_handle *trans, + const struct btrfs_log_ctx *ctx, + u64 ino, u64 parent), + + TP_ARGS(trans, ctx, ino, parent), + + TP_STRUCT__entry_btrfs( + __field( u64, root_objectid ) + __field( u64, transid ) + __field( u64, ctx_ino ) + __field( u64, conflict_ino ) + __field( u64, conflict_ino_parent ) + ), + + TP_fast_assign( + TP_fast_assign_fsid(trans->fs_info); + __entry->root_objectid = btrfs_root_id(ctx->inode->root); + __entry->transid = trans->transid; + __entry->ctx_ino = btrfs_ino(ctx->inode); + __entry->conflict_ino = ino; + __entry->conflict_ino_parent = parent; + ), + + TP_printk_btrfs("root=%llu(%s) transid=%llu ctx_ino=%llu conflict_ino=%llu" + " conflict_ino_parent=%llu", + show_root_type(__entry->root_objectid), __entry->transid, + __entry->ctx_ino, __entry->conflict_ino, + __entry->conflict_ino_parent) +); + +TRACE_EVENT(btrfs_add_conflicting_inode_exit, + + TP_PROTO(const struct btrfs_trans_handle *trans, + const struct btrfs_log_ctx *ctx, + u64 ino, u64 parent, int ret), + + TP_ARGS(trans, ctx, ino, parent, ret), + + TP_STRUCT__entry_btrfs( + __field( u64, root_objectid ) + __field( u64, transid ) + __field( u64, ctx_ino ) + __field( u64, conflict_ino ) + __field( u64, conflict_ino_parent ) + __field( int, ret ) + ), + + TP_fast_assign( + TP_fast_assign_fsid(trans->fs_info); + __entry->root_objectid = btrfs_root_id(ctx->inode->root); + __entry->transid = trans->transid; + __entry->ctx_ino = btrfs_ino(ctx->inode); + __entry->conflict_ino = ino; + __entry->conflict_ino_parent = parent; + __entry->ret = ret; + ), + + TP_printk_btrfs("root=%llu(%s) transid=%llu ctx_ino=%llu conflict_ino=%llu" + " conflict_ino_parent=%llu ret=%d", + show_root_type(__entry->root_objectid), __entry->transid, + __entry->ctx_ino, __entry->conflict_ino, + __entry->conflict_ino_parent, __entry->ret) +); + +TRACE_EVENT(btrfs_log_conflicting_inodes_enter, + + TP_PROTO(const struct btrfs_trans_handle *trans, + const struct btrfs_log_ctx *ctx), + + TP_ARGS(trans, ctx), + + TP_STRUCT__entry_btrfs( + __field( u64, root_objectid ) + __field( u64, transid ) + __field( u64, ctx_ino ) + ), + + TP_fast_assign( + TP_fast_assign_fsid(trans->fs_info); + __entry->root_objectid = btrfs_root_id(ctx->inode->root); + __entry->transid = trans->transid; + __entry->ctx_ino = btrfs_ino(ctx->inode); + ), + + TP_printk_btrfs("root=%llu(%s) transid=%llu ctx_ino=%llu", + show_root_type(__entry->root_objectid), __entry->transid, + __entry->ctx_ino) +); + +TRACE_EVENT(btrfs_log_conflicting_inodes_exit, + + TP_PROTO(const struct btrfs_trans_handle *trans, + const struct btrfs_log_ctx *ctx, + int ret), + + TP_ARGS(trans, ctx, ret), + + TP_STRUCT__entry_btrfs( + __field( u64, root_objectid ) + __field( u64, transid ) + __field( u64, ctx_ino ) + __field( int, ret ) + ), + + TP_fast_assign( + TP_fast_assign_fsid(trans->fs_info); + __entry->root_objectid = btrfs_root_id(ctx->inode->root); + __entry->transid = trans->transid; + __entry->ctx_ino = btrfs_ino(ctx->inode); + __entry->ret = ret; + ), + + TP_printk_btrfs("root=%llu(%s) transid=%llu ctx_ino=%llu ret=%d", + show_root_type(__entry->root_objectid), __entry->transid, + __entry->ctx_ino, __entry->ret) +); + +TRACE_EVENT(btrfs_log_new_delayed_dentries_enter, + + TP_PROTO(const struct btrfs_trans_handle *trans, + const struct btrfs_inode *inode), + + TP_ARGS(trans, inode), + + TP_STRUCT__entry_btrfs( + __field( u64, root_objectid ) + __field( u64, transid ) + __field( u64, ino ) + ), + + TP_fast_assign( + TP_fast_assign_fsid(trans->fs_info); + __entry->root_objectid = btrfs_root_id(inode->root); + __entry->transid = trans->transid; + __entry->ino = btrfs_ino(inode); + ), + + TP_printk_btrfs("root=%llu(%s) transid=%llu ino=%llu", + show_root_type(__entry->root_objectid), __entry->transid, + __entry->ino) +); + +TRACE_EVENT(btrfs_log_new_delayed_dentries_exit, + + TP_PROTO(const struct btrfs_trans_handle *trans, + const struct btrfs_inode *inode, + int ret), + + TP_ARGS(trans, inode, ret), + + TP_STRUCT__entry_btrfs( + __field( u64, root_objectid ) + __field( u64, transid ) + __field( u64, ino ) + __field( int, ret ) + ), + + TP_fast_assign( + TP_fast_assign_fsid(trans->fs_info); + __entry->root_objectid = btrfs_root_id(inode->root); + __entry->transid = trans->transid; + __entry->ino = btrfs_ino(inode); + __entry->ret = ret; + ), + + TP_printk_btrfs("root=%llu(%s) transid=%llu ino=%llu ret=%d", + show_root_type(__entry->root_objectid), __entry->transid, + __entry->ino, __entry->ret) +); + +TRACE_EVENT(btrfs_record_unlink_dir, + + TP_PROTO(const struct btrfs_trans_handle *trans, + const struct btrfs_inode *dir, + const struct btrfs_inode *inode, + bool for_rename), + + TP_ARGS(trans, dir, inode, for_rename), + + TP_STRUCT__entry_btrfs( + __field( u64, root_objectid ) + __field( u64, transid ) + __field( u64, ino ) + __field( u64, dir ) + __field( bool, for_rename ) + ), + + TP_fast_assign( + TP_fast_assign_fsid(trans->fs_info); + __entry->root_objectid = btrfs_root_id(inode->root); + __entry->transid = trans->transid; + __entry->ino = btrfs_ino(inode); + __entry->dir = btrfs_ino(dir); + __entry->for_rename = for_rename; + ), + + TP_printk_btrfs("root=%llu(%s) transid=%llu ino=%llu dir=%llu for_rename=%d", + show_root_type(__entry->root_objectid), __entry->transid, + __entry->ino, __entry->dir, __entry->for_rename) +); + +TRACE_EVENT(btrfs_record_snapshot_destroy, + + TP_PROTO(const struct btrfs_trans_handle *trans, + const struct btrfs_inode *dir), + + TP_ARGS(trans, dir), + + TP_STRUCT__entry_btrfs( + __field( u64, root_objectid ) + __field( u64, transid ) + __field( u64, dir ) + ), + + TP_fast_assign( + TP_fast_assign_fsid(trans->fs_info); + __entry->root_objectid = btrfs_root_id(dir->root); + __entry->transid = trans->transid; + __entry->dir = btrfs_ino(dir); + ), + + TP_printk_btrfs("root=%llu(%s) transid=%llu dir=%llu", + show_root_type(__entry->root_objectid), __entry->transid, + __entry->dir) +); + +TRACE_EVENT(btrfs_record_new_subvolume, + + TP_PROTO(const struct btrfs_trans_handle *trans, + const struct btrfs_inode *dir), + + TP_ARGS(trans, dir), + + TP_STRUCT__entry_btrfs( + __field( u64, root_objectid ) + __field( u64, transid ) + __field( u64, dir ) + ), + + TP_fast_assign( + TP_fast_assign_fsid(trans->fs_info); + __entry->root_objectid = btrfs_root_id(dir->root); + __entry->transid = trans->transid; + __entry->dir = btrfs_ino(dir); + ), + + TP_printk_btrfs("root=%llu(%s) transid=%llu dir=%llu", + show_root_type(__entry->root_objectid), __entry->transid, + __entry->dir) +); + +TRACE_EVENT(btrfs_log_new_name_enter, + + TP_PROTO(const struct btrfs_trans_handle *trans, + const struct btrfs_inode *inode, + const struct btrfs_inode *old_dir, + u64 old_dir_index), + + TP_ARGS(trans, inode, old_dir, old_dir_index), + + TP_STRUCT__entry_btrfs( + __field( u64, root_objectid ) + __field( u64, transid ) + __field( u64, ino ) + __field( umode_t, mode ) + __field( u64, old_dir_ino ) + __field( u64, old_dir_index ) + ), + + TP_fast_assign( + TP_fast_assign_fsid(trans->fs_info); + __entry->root_objectid = btrfs_root_id(inode->root); + __entry->transid = trans->transid; + __entry->ino = btrfs_ino(inode); + __entry->mode = inode->vfs_inode.i_mode; + __entry->old_dir_ino = old_dir ? btrfs_ino(old_dir) : 0; + __entry->old_dir_index = old_dir_index; + ), + + TP_printk_btrfs("root=%llu(%s) transid=%llu ino=%llu type=%s" + " old_dir=%llu old_dir_index=%llu", + show_root_type(__entry->root_objectid), __entry->transid, + __entry->ino, show_inode_type(__entry->mode), + __entry->old_dir_ino, __entry->old_dir_index) +); + +TRACE_EVENT(btrfs_log_new_name_exit, + + TP_PROTO(const struct btrfs_trans_handle *trans, + const struct btrfs_inode *inode, + const struct btrfs_inode *old_dir, + int ret), + + TP_ARGS(trans, inode, old_dir, ret), + + TP_STRUCT__entry_btrfs( + __field( u64, root_objectid ) + __field( u64, transid ) + __field( u64, ino ) + __field( u64, old_dir_ino ) + __field( int, ret ) + ), + + TP_fast_assign( + TP_fast_assign_fsid(trans->fs_info); + __entry->root_objectid = btrfs_root_id(inode->root); + __entry->transid = trans->transid; + __entry->ino = btrfs_ino(inode); + __entry->old_dir_ino = old_dir ? btrfs_ino(old_dir) : 0; + __entry->ret = ret; + ), + + TP_printk_btrfs("root=%llu(%s) transid=%llu ino=%llu old_dir=%llu ret=%d", + show_root_type(__entry->root_objectid), __entry->transid, + __entry->ino, __entry->old_dir_ino, __entry->ret) +); + +/* Ideally call this while under root->log_mutex (but not always possible). */ +TRACE_EVENT(btrfs_sync_log_enter, + + TP_PROTO(const struct btrfs_trans_handle *trans, + const struct btrfs_root *root, + const struct btrfs_log_ctx *ctx), + + TP_ARGS(trans, root, ctx), + + TP_STRUCT__entry_btrfs( + __field( u64, root_objectid ) + __field( u64, transid ) + __field( int, ctx_log_transid ) + __field( int, root_log_transid ) + __field( int, log_transid_committed ) + __field( bool, log_committing ) + __field( bool, log_committing_prev ) + __field( int, log_writers ) + ), + + TP_fast_assign( + TP_fast_assign_fsid(trans->fs_info); + __entry->root_objectid = btrfs_root_id(root); + __entry->transid = trans->transid; + __entry->ctx_log_transid = ctx->log_transid; + __entry->root_log_transid = btrfs_get_root_log_transid(root); + __entry->log_transid_committed = + data_race(root->log_transid_committed); + __entry->log_committing = + atomic_read(&root->log_commit[ctx->log_transid % 2]); + __entry->log_committing_prev = + atomic_read(&root->log_commit[(ctx->log_transid + 1) % 2]); + __entry->log_writers = atomic_read(&root->log_writers); + ), + + TP_printk_btrfs("root=%llu(%s) transid=%llu ctx_log_transid=%d" + " root_log_transid=%d log_transid_committed=%d" + " log_committing=%d log_committing_prev=%d log_writers=%d", + show_root_type(__entry->root_objectid), __entry->transid, + __entry->ctx_log_transid, __entry->root_log_transid, + __entry->log_transid_committed, __entry->log_committing, + __entry->log_committing_prev, __entry->log_writers) +); + +/* + * Ideally call this while under root->log_mutex and in the same critical + * section that calls the btrfs_sync_log_enter() trace event (though it's not + * always possible). + */ +TRACE_EVENT(btrfs_sync_log_exit, + + TP_PROTO(const struct btrfs_trans_handle *trans, + const struct btrfs_root *root, + const struct btrfs_log_ctx *ctx, + int ret), + + TP_ARGS(trans, root, ctx, ret), + + TP_STRUCT__entry_btrfs( + __field( u64, root_objectid ) + __field( u64, transid ) + __field( int, ctx_log_transid ) + __field( int, root_log_transid ) + __field( int, log_transid_committed ) + __field( int, ret ) + ), + + TP_fast_assign( + TP_fast_assign_fsid(trans->fs_info); + __entry->root_objectid = btrfs_root_id(root); + __entry->transid = trans->transid; + __entry->ctx_log_transid = ctx->log_transid; + __entry->root_log_transid = btrfs_get_root_log_transid(root); + __entry->log_transid_committed = + data_race(root->log_transid_committed); + __entry->ret = ret; + ), + + TP_printk_btrfs("root=%llu(%s) transid=%llu ctx_log_transid=%d" + " root_log_transid=%d log_transid_committed=%d ret=%d", + show_root_type(__entry->root_objectid), __entry->transid, + __entry->ctx_log_transid, __entry->root_log_transid, + __entry->log_transid_committed, __entry->ret) ); TRACE_EVENT(btrfs_sync_fs, diff --git a/include/uapi/linux/btrfs.h b/include/uapi/linux/btrfs.h index 9165154a274d..9b576603b3f1 100644 --- a/include/uapi/linux/btrfs.h +++ b/include/uapi/linux/btrfs.h @@ -1100,6 +1100,38 @@ enum btrfs_err_code { BTRFS_ERROR_DEV_RAID1C4_MIN_NOT_MET, }; +/* Flags for struct btrfs_ioctl_get_csums_entry::type. */ +#define BTRFS_GET_CSUMS_HAS_CSUMS (1U << 0) +#define BTRFS_GET_CSUMS_ZEROED (1U << 1) +#define BTRFS_GET_CSUMS_NODATASUM (1U << 2) +#define BTRFS_GET_CSUMS_COMPRESSED (1U << 3) +#define BTRFS_GET_CSUMS_ENCRYPTED (1U << 4) +#define BTRFS_GET_CSUMS_INLINE (1U << 5) + +struct btrfs_ioctl_get_csums_entry { + /* File offset of this range. */ + __u64 offset; + /* Length in bytes. */ + __u64 length; + /* One of BTRFS_GET_CSUMS_* types. */ + __u32 type; + /* Padding, must be 0. */ + __u32 reserved; +}; + +struct btrfs_ioctl_get_csums_args { + /* In/out: file offset in bytes. */ + __u64 offset; + /* In/out: range length in bytes. */ + __u64 length; + /* In/out: buffer capacity / bytes written. */ + __u64 buf_size; + /* In: flags, must be 0 for now. */ + __u64 flags; + /* Out: entries of type btrfs_ioctl_get_csums_entry + csum data */ + __u8 buf[]; +}; + /* Flags for IOC_SHUTDOWN, must match XFS_FSOP_GOING_FLAGS_* flags. */ #define BTRFS_SHUTDOWN_FLAGS_DEFAULT 0x0 #define BTRFS_SHUTDOWN_FLAGS_LOGFLUSH 0x1 @@ -1226,6 +1258,8 @@ enum btrfs_err_code { struct btrfs_ioctl_encoded_io_args) #define BTRFS_IOC_SUBVOL_SYNC_WAIT _IOW(BTRFS_IOCTL_MAGIC, 65, \ struct btrfs_ioctl_subvol_wait) +#define BTRFS_IOC_GET_CSUMS _IOWR(BTRFS_IOCTL_MAGIC, 66, \ + struct btrfs_ioctl_get_csums_args) /* Shutdown ioctl should follow XFS's interfaces, thus not using btrfs magic. */ #define BTRFS_IOC_SHUTDOWN _IOR('X', 125, __u32) diff --git a/lib/crc/Kconfig b/lib/crc/Kconfig index f47bb4c706fb..927fc6a6b2b9 100644 --- a/lib/crc/Kconfig +++ b/lib/crc/Kconfig @@ -82,6 +82,7 @@ config CRC64 config CRC64_ARCH bool depends on CRC64 && CRC_OPTIMIZATIONS + default y if ARM && KERNEL_MODE_NEON && !CPU_BIG_ENDIAN default y if ARM64 default y if RISCV && RISCV_ISA_ZBC && 64BIT default y if X86_64 diff --git a/lib/crc/Makefile b/lib/crc/Makefile index ff213590e4e3..386e9c175263 100644 --- a/lib/crc/Makefile +++ b/lib/crc/Makefile @@ -39,9 +39,12 @@ crc64-y := crc64-main.o ifeq ($(CONFIG_CRC64_ARCH),y) CFLAGS_crc64-main.o += -I$(src)/$(SRCARCH) -CFLAGS_REMOVE_arm64/crc64-neon-inner.o += $(CC_FLAGS_NO_FPU) -CFLAGS_arm64/crc64-neon-inner.o += $(CC_FLAGS_FPU) -march=armv8-a+crypto -crc64-$(CONFIG_ARM64) += arm64/crc64-neon-inner.o +crc64-cflags-$(CONFIG_ARM) += -march=armv8-a -mfpu=crypto-neon-fp-armv8 +crc64-cflags-$(CONFIG_ARM64) += -march=armv8-a+crypto +CFLAGS_REMOVE_crc64-neon.o += $(CC_FLAGS_NO_FPU) +CFLAGS_crc64-neon.o += $(CC_FLAGS_FPU) -I$(src)/$(SRCARCH) $(crc64-cflags-y) +crc64-$(CONFIG_ARM) += crc64-neon.o +crc64-$(CONFIG_ARM64) += crc64-neon.o crc64-$(CONFIG_RISCV) += riscv/crc64_lsb.o riscv/crc64_msb.o crc64-$(CONFIG_X86) += x86/crc64-pclmul.o diff --git a/lib/crc/arm/crc64-neon.h b/lib/crc/arm/crc64-neon.h new file mode 100644 index 000000000000..645f553220ff --- /dev/null +++ b/lib/crc/arm/crc64-neon.h @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0-only + +static inline uint64x2_t pmull64(uint64x2_t a, uint64x2_t b) +{ + uint64_t l = vgetq_lane_u64(a, 0); + uint64_t m = vgetq_lane_u64(b, 0); + uint64x2_t result; + + asm("vmull.p64 %q0, %P1, %P2" : "=w"(result) : "w"(l), "w"(m)); + + return result; +} + +static inline uint64x2_t pmull64_high(uint64x2_t a, uint64x2_t b) +{ + uint64_t l = vgetq_lane_u64(a, 1); + uint64_t m = vgetq_lane_u64(b, 1); + uint64x2_t result; + + asm("vmull.p64 %q0, %P1, %P2" : "=w"(result) : "w"(l), "w"(m)); + + return result; +} + +static inline uint64x2_t pmull64_hi_lo(uint64x2_t a, uint64x2_t b) +{ + uint64_t l = vgetq_lane_u64(a, 1); + uint64_t m = vgetq_lane_u64(b, 0); + uint64x2_t result; + + asm("vmull.p64 %q0, %P1, %P2" : "=w"(result) : "w"(l), "w"(m)); + + return result; +} diff --git a/lib/crc/arm/crc64.h b/lib/crc/arm/crc64.h new file mode 100644 index 000000000000..de274288af61 --- /dev/null +++ b/lib/crc/arm/crc64.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * CRC64 using ARM PMULL instructions + */ + +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); + +u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len); + +#define crc64_be_arch crc64_be_generic + +static inline u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len) +{ + if (len >= 128 && static_branch_likely(&have_pmull) && + likely(may_use_simd())) { + do { + size_t chunk = min_t(size_t, len & ~15, SZ_4K); + + scoped_ksimd() + crc = crc64_nvme_neon(crc, p, chunk); + + p += chunk; + len -= chunk; + } while (len >= 128); + } + return crc64_nvme_generic(crc, p, len); +} + +#define crc64_mod_init_arch crc64_mod_init_arch +static void crc64_mod_init_arch(void) +{ + if (elf_hwcap2 & HWCAP2_PMULL) + static_branch_enable(&have_pmull); +} diff --git a/lib/crc/arm64/crc64-neon.h b/lib/crc/arm64/crc64-neon.h new file mode 100644 index 000000000000..fcd5b1e6f812 --- /dev/null +++ b/lib/crc/arm64/crc64-neon.h @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0-only + +static inline uint64x2_t pmull64(uint64x2_t a, uint64x2_t b) +{ + return vreinterpretq_u64_p128(vmull_p64(vgetq_lane_u64(a, 0), + vgetq_lane_u64(b, 0))); +} + +static inline uint64x2_t pmull64_high(uint64x2_t a, uint64x2_t b) +{ + poly64x2_t l = vreinterpretq_p64_u64(a); + poly64x2_t m = vreinterpretq_p64_u64(b); + + return vreinterpretq_u64_p128(vmull_high_p64(l, m)); +} + +static inline uint64x2_t pmull64_hi_lo(uint64x2_t a, uint64x2_t b) +{ + return vreinterpretq_u64_p128(vmull_p64(vgetq_lane_u64(a, 1), + vgetq_lane_u64(b, 0))); +} diff --git a/lib/crc/arm64/crc64.h b/lib/crc/arm64/crc64.h index 60151ec3035a..c7a69e1f3d8f 100644 --- a/lib/crc/arm64/crc64.h +++ b/lib/crc/arm64/crc64.h @@ -8,7 +8,7 @@ #include #include -u64 crc64_nvme_arm64_c(u64 crc, const u8 *p, size_t len); +u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len); #define crc64_be_arch crc64_be_generic @@ -19,7 +19,7 @@ static inline u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len) size_t chunk = len & ~15; scoped_ksimd() - crc = crc64_nvme_arm64_c(crc, p, chunk); + crc = crc64_nvme_neon(crc, p, chunk); p += chunk; len &= 15; diff --git a/lib/crc/arm64/crc64-neon-inner.c b/lib/crc/crc64-neon.c similarity index 62% rename from lib/crc/arm64/crc64-neon-inner.c rename to lib/crc/crc64-neon.c index 28527e544ff6..4753fb94a4be 100644 --- a/lib/crc/arm64/crc64-neon-inner.c +++ b/lib/crc/crc64-neon.c @@ -6,7 +6,9 @@ #include #include -u64 crc64_nvme_arm64_c(u64 crc, const u8 *p, size_t len); +#include "crc64-neon.h" + +u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len); /* x^191 mod G, x^127 mod G */ static const u64 fold_consts_val[2] = { 0xeadc41fd2ba3d420ULL, @@ -15,27 +17,7 @@ static const u64 fold_consts_val[2] = { 0xeadc41fd2ba3d420ULL, static const u64 bconsts_val[2] = { 0x27ecfa329aef9f77ULL, 0x34d926535897936aULL }; -static inline uint64x2_t pmull64(uint64x2_t a, uint64x2_t b) -{ - return vreinterpretq_u64_p128(vmull_p64(vgetq_lane_u64(a, 0), - vgetq_lane_u64(b, 0))); -} - -static inline uint64x2_t pmull64_high(uint64x2_t a, uint64x2_t b) -{ - poly64x2_t l = vreinterpretq_p64_u64(a); - poly64x2_t m = vreinterpretq_p64_u64(b); - - return vreinterpretq_u64_p128(vmull_high_p64(l, m)); -} - -static inline uint64x2_t pmull64_hi_lo(uint64x2_t a, uint64x2_t b) -{ - return vreinterpretq_u64_p128(vmull_p64(vgetq_lane_u64(a, 1), - vgetq_lane_u64(b, 0))); -} - -u64 crc64_nvme_arm64_c(u64 crc, const u8 *p, size_t len) +u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len) { uint64x2_t fold_consts = vld1q_u64(fold_consts_val); uint64x2_t v0 = { crc, 0 }; diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index d3904b72dae7..591c1c2a7fb3 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -131,11 +131,6 @@ config CRYPTO_LIB_MD5 The MD5 and HMAC-MD5 library functions. Select this if your module uses any of the functions from . -config CRYPTO_LIB_MD5_ARCH - bool - depends on CRYPTO_LIB_MD5 && !UML - default y if PPC - config CRYPTO_LIB_MLDSA tristate select CRYPTO_LIB_SHA3 diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 4ad91f390038..f1e9bf89785f 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -187,10 +187,6 @@ clean-files += powerpc/ghashp8-ppc.S obj-$(CONFIG_CRYPTO_LIB_MD5) += libmd5.o libmd5-y := md5.o -ifeq ($(CONFIG_CRYPTO_LIB_MD5_ARCH),y) -CFLAGS_md5.o += -I$(src)/$(SRCARCH) -libmd5-$(CONFIG_PPC) += powerpc/md5-asm.o -endif # CONFIG_CRYPTO_LIB_MD5_ARCH ################################################################################ diff --git a/lib/crypto/gf128hash.c b/lib/crypto/gf128hash.c index 2650603d8ba8..8dcdf5ec98be 100644 --- a/lib/crypto/gf128hash.c +++ b/lib/crypto/gf128hash.c @@ -109,7 +109,7 @@ static void clmul64(u64 a, u64 b, u64 *out_lo, u64 *out_hi) #else /* CONFIG_ARCH_SUPPORTS_INT128 */ /* Do a 32 x 32 => 64 bit carryless multiplication. */ -static u64 clmul32(u32 a, u32 b) +static noinline_for_stack u64 clmul32(u32 a, u32 b) { /* * With 32-bit multiplicands and one term every 4 bits, there are up to diff --git a/lib/crypto/md5.c b/lib/crypto/md5.c index c4af57db0ea8..6bf130cfbbf9 100644 --- a/lib/crypto/md5.c +++ b/lib/crypto/md5.c @@ -2,8 +2,8 @@ /* * MD5 and HMAC-MD5 library functions * - * md5_block_generic() is derived from cryptoapi implementation, originally - * based on the public domain implementation written by Colin Plumb in 1993. + * md5_block() is derived from cryptoapi implementation, originally based on the + * public domain implementation written by Colin Plumb in 1993. * * Copyright (c) Cryptoapi developers. * Copyright (c) 2002 James Morris @@ -31,8 +31,8 @@ static const struct md5_block_state md5_iv = { #define MD5STEP(f, w, x, y, z, in, s) \ (w += f(x, y, z) + in, w = rol32(w, s) + x) -static void md5_block_generic(struct md5_block_state *state, - const u8 data[MD5_BLOCK_SIZE]) +static void md5_block(struct md5_block_state *state, + const u8 data[MD5_BLOCK_SIZE]) { u32 in[MD5_BLOCK_WORDS]; u32 a, b, c, d; @@ -119,21 +119,15 @@ static void md5_block_generic(struct md5_block_state *state, state->h[3] += d; } -static void __maybe_unused md5_blocks_generic(struct md5_block_state *state, - const u8 *data, size_t nblocks) +static void md5_blocks(struct md5_block_state *state, + const u8 *data, size_t nblocks) { do { - md5_block_generic(state, data); + md5_block(state, data); data += MD5_BLOCK_SIZE; } while (--nblocks); } -#ifdef CONFIG_CRYPTO_LIB_MD5_ARCH -#include "md5.h" /* $(SRCARCH)/md5.h */ -#else -#define md5_blocks md5_blocks_generic -#endif - void md5_init(struct md5_ctx *ctx) { ctx->state = md5_iv; diff --git a/lib/crypto/powerpc/md5-asm.S b/lib/crypto/powerpc/md5-asm.S deleted file mode 100644 index fa6bc440cf4a..000000000000 --- a/lib/crypto/powerpc/md5-asm.S +++ /dev/null @@ -1,235 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Fast MD5 implementation for PPC - * - * Copyright (c) 2015 Markus Stockhausen - */ -#include -#include -#include - -#define rHP r3 -#define rWP r4 - -#define rH0 r0 -#define rH1 r6 -#define rH2 r7 -#define rH3 r5 - -#define rW00 r8 -#define rW01 r9 -#define rW02 r10 -#define rW03 r11 -#define rW04 r12 -#define rW05 r14 -#define rW06 r15 -#define rW07 r16 -#define rW08 r17 -#define rW09 r18 -#define rW10 r19 -#define rW11 r20 -#define rW12 r21 -#define rW13 r22 -#define rW14 r23 -#define rW15 r24 - -#define rT0 r25 -#define rT1 r26 - -#define INITIALIZE \ - PPC_STLU r1,-INT_FRAME_SIZE(r1); \ - SAVE_GPRS(14, 26, r1) /* push registers onto stack */ - -#define FINALIZE \ - REST_GPRS(14, 26, r1); /* pop registers from stack */ \ - addi r1,r1,INT_FRAME_SIZE - -#ifdef __BIG_ENDIAN__ -#define LOAD_DATA(reg, off) \ - lwbrx reg,0,rWP; /* load data */ -#define INC_PTR \ - addi rWP,rWP,4; /* increment per word */ -#define NEXT_BLOCK /* nothing to do */ -#else -#define LOAD_DATA(reg, off) \ - lwz reg,off(rWP); /* load data */ -#define INC_PTR /* nothing to do */ -#define NEXT_BLOCK \ - addi rWP,rWP,64; /* increment per block */ -#endif - -#define R_00_15(a, b, c, d, w0, w1, p, q, off, k0h, k0l, k1h, k1l) \ - LOAD_DATA(w0, off) /* W */ \ - and rT0,b,c; /* 1: f = b and c */ \ - INC_PTR /* ptr++ */ \ - andc rT1,d,b; /* 1: f' = ~b and d */ \ - LOAD_DATA(w1, off+4) /* W */ \ - or rT0,rT0,rT1; /* 1: f = f or f' */ \ - addi w0,w0,k0l; /* 1: wk = w + k */ \ - add a,a,rT0; /* 1: a = a + f */ \ - addis w0,w0,k0h; /* 1: wk = w + k' */ \ - addis w1,w1,k1h; /* 2: wk = w + k */ \ - add a,a,w0; /* 1: a = a + wk */ \ - addi w1,w1,k1l; /* 2: wk = w + k' */ \ - rotrwi a,a,p; /* 1: a = a rotl x */ \ - add d,d,w1; /* 2: a = a + wk */ \ - add a,a,b; /* 1: a = a + b */ \ - and rT0,a,b; /* 2: f = b and c */ \ - andc rT1,c,a; /* 2: f' = ~b and d */ \ - or rT0,rT0,rT1; /* 2: f = f or f' */ \ - add d,d,rT0; /* 2: a = a + f */ \ - INC_PTR /* ptr++ */ \ - rotrwi d,d,q; /* 2: a = a rotl x */ \ - add d,d,a; /* 2: a = a + b */ - -#define R_16_31(a, b, c, d, w0, w1, p, q, k0h, k0l, k1h, k1l) \ - andc rT0,c,d; /* 1: f = c and ~d */ \ - and rT1,b,d; /* 1: f' = b and d */ \ - addi w0,w0,k0l; /* 1: wk = w + k */ \ - or rT0,rT0,rT1; /* 1: f = f or f' */ \ - addis w0,w0,k0h; /* 1: wk = w + k' */ \ - add a,a,rT0; /* 1: a = a + f */ \ - addi w1,w1,k1l; /* 2: wk = w + k */ \ - add a,a,w0; /* 1: a = a + wk */ \ - addis w1,w1,k1h; /* 2: wk = w + k' */ \ - andc rT0,b,c; /* 2: f = c and ~d */ \ - rotrwi a,a,p; /* 1: a = a rotl x */ \ - add a,a,b; /* 1: a = a + b */ \ - add d,d,w1; /* 2: a = a + wk */ \ - and rT1,a,c; /* 2: f' = b and d */ \ - or rT0,rT0,rT1; /* 2: f = f or f' */ \ - add d,d,rT0; /* 2: a = a + f */ \ - rotrwi d,d,q; /* 2: a = a rotl x */ \ - add d,d,a; /* 2: a = a +b */ - -#define R_32_47(a, b, c, d, w0, w1, p, q, k0h, k0l, k1h, k1l) \ - xor rT0,b,c; /* 1: f' = b xor c */ \ - addi w0,w0,k0l; /* 1: wk = w + k */ \ - xor rT1,rT0,d; /* 1: f = f xor f' */ \ - addis w0,w0,k0h; /* 1: wk = w + k' */ \ - add a,a,rT1; /* 1: a = a + f */ \ - addi w1,w1,k1l; /* 2: wk = w + k */ \ - add a,a,w0; /* 1: a = a + wk */ \ - addis w1,w1,k1h; /* 2: wk = w + k' */ \ - rotrwi a,a,p; /* 1: a = a rotl x */ \ - add d,d,w1; /* 2: a = a + wk */ \ - add a,a,b; /* 1: a = a + b */ \ - xor rT1,rT0,a; /* 2: f = b xor f' */ \ - add d,d,rT1; /* 2: a = a + f */ \ - rotrwi d,d,q; /* 2: a = a rotl x */ \ - add d,d,a; /* 2: a = a + b */ - -#define R_48_63(a, b, c, d, w0, w1, p, q, k0h, k0l, k1h, k1l) \ - addi w0,w0,k0l; /* 1: w = w + k */ \ - orc rT0,b,d; /* 1: f = b or ~d */ \ - addis w0,w0,k0h; /* 1: w = w + k' */ \ - xor rT0,rT0,c; /* 1: f = f xor c */ \ - add a,a,w0; /* 1: a = a + wk */ \ - addi w1,w1,k1l; /* 2: w = w + k */ \ - add a,a,rT0; /* 1: a = a + f */ \ - addis w1,w1,k1h; /* 2: w = w + k' */ \ - rotrwi a,a,p; /* 1: a = a rotl x */ \ - add a,a,b; /* 1: a = a + b */ \ - orc rT0,a,c; /* 2: f = b or ~d */ \ - add d,d,w1; /* 2: a = a + wk */ \ - xor rT0,rT0,b; /* 2: f = f xor c */ \ - add d,d,rT0; /* 2: a = a + f */ \ - rotrwi d,d,q; /* 2: a = a rotl x */ \ - add d,d,a; /* 2: a = a + b */ - -_GLOBAL(ppc_md5_transform) - INITIALIZE - - mtctr r5 - lwz rH0,0(rHP) - lwz rH1,4(rHP) - lwz rH2,8(rHP) - lwz rH3,12(rHP) - -ppc_md5_main: - R_00_15(rH0, rH1, rH2, rH3, rW00, rW01, 25, 20, 0, - 0xd76b, -23432, 0xe8c8, -18602) - R_00_15(rH2, rH3, rH0, rH1, rW02, rW03, 15, 10, 8, - 0x2420, 0x70db, 0xc1be, -12562) - R_00_15(rH0, rH1, rH2, rH3, rW04, rW05, 25, 20, 16, - 0xf57c, 0x0faf, 0x4788, -14806) - R_00_15(rH2, rH3, rH0, rH1, rW06, rW07, 15, 10, 24, - 0xa830, 0x4613, 0xfd47, -27391) - R_00_15(rH0, rH1, rH2, rH3, rW08, rW09, 25, 20, 32, - 0x6981, -26408, 0x8b45, -2129) - R_00_15(rH2, rH3, rH0, rH1, rW10, rW11, 15, 10, 40, - 0xffff, 0x5bb1, 0x895d, -10306) - R_00_15(rH0, rH1, rH2, rH3, rW12, rW13, 25, 20, 48, - 0x6b90, 0x1122, 0xfd98, 0x7193) - R_00_15(rH2, rH3, rH0, rH1, rW14, rW15, 15, 10, 56, - 0xa679, 0x438e, 0x49b4, 0x0821) - - R_16_31(rH0, rH1, rH2, rH3, rW01, rW06, 27, 23, - 0x0d56, 0x6e0c, 0x1810, 0x6d2d) - R_16_31(rH2, rH3, rH0, rH1, rW11, rW00, 18, 12, - 0x9d02, -32109, 0x124c, 0x2332) - R_16_31(rH0, rH1, rH2, rH3, rW05, rW10, 27, 23, - 0x8ea7, 0x4a33, 0x0245, -18270) - R_16_31(rH2, rH3, rH0, rH1, rW15, rW04, 18, 12, - 0x8eee, -8608, 0xf258, -5095) - R_16_31(rH0, rH1, rH2, rH3, rW09, rW14, 27, 23, - 0x969d, -10697, 0x1cbe, -15288) - R_16_31(rH2, rH3, rH0, rH1, rW03, rW08, 18, 12, - 0x3317, 0x3e99, 0xdbd9, 0x7c15) - R_16_31(rH0, rH1, rH2, rH3, rW13, rW02, 27, 23, - 0xac4b, 0x7772, 0xd8cf, 0x331d) - R_16_31(rH2, rH3, rH0, rH1, rW07, rW12, 18, 12, - 0x6a28, 0x6dd8, 0x219a, 0x3b68) - - R_32_47(rH0, rH1, rH2, rH3, rW05, rW08, 28, 21, - 0x29cb, 0x28e5, 0x4218, -7788) - R_32_47(rH2, rH3, rH0, rH1, rW11, rW14, 16, 9, - 0x473f, 0x06d1, 0x3aae, 0x3036) - R_32_47(rH0, rH1, rH2, rH3, rW01, rW04, 28, 21, - 0xaea1, -15134, 0x640b, -11295) - R_32_47(rH2, rH3, rH0, rH1, rW07, rW10, 16, 9, - 0x8f4c, 0x4887, 0xbc7c, -22499) - R_32_47(rH0, rH1, rH2, rH3, rW13, rW00, 28, 21, - 0x7eb8, -27199, 0x00ea, 0x6050) - R_32_47(rH2, rH3, rH0, rH1, rW03, rW06, 16, 9, - 0xe01a, 0x22fe, 0x4447, 0x69c5) - R_32_47(rH0, rH1, rH2, rH3, rW09, rW12, 28, 21, - 0xb7f3, 0x0253, 0x59b1, 0x4d5b) - R_32_47(rH2, rH3, rH0, rH1, rW15, rW02, 16, 9, - 0x4701, -27017, 0xc7bd, -19859) - - R_48_63(rH0, rH1, rH2, rH3, rW00, rW07, 26, 22, - 0x0988, -1462, 0x4c70, -19401) - R_48_63(rH2, rH3, rH0, rH1, rW14, rW05, 17, 11, - 0xadaf, -5221, 0xfc99, 0x66f7) - R_48_63(rH0, rH1, rH2, rH3, rW12, rW03, 26, 22, - 0x7e80, -16418, 0xba1e, -25587) - R_48_63(rH2, rH3, rH0, rH1, rW10, rW01, 17, 11, - 0x4130, 0x380d, 0xe0c5, 0x738d) - lwz rW00,0(rHP) - R_48_63(rH0, rH1, rH2, rH3, rW08, rW15, 26, 22, - 0xe837, -30770, 0xde8a, 0x69e8) - lwz rW14,4(rHP) - R_48_63(rH2, rH3, rH0, rH1, rW06, rW13, 17, 11, - 0x9e79, 0x260f, 0x256d, -27941) - lwz rW12,8(rHP) - R_48_63(rH0, rH1, rH2, rH3, rW04, rW11, 26, 22, - 0xab75, -20775, 0x4f9e, -28397) - lwz rW10,12(rHP) - R_48_63(rH2, rH3, rH0, rH1, rW02, rW09, 17, 11, - 0x662b, 0x7c56, 0x11b2, 0x0358) - - add rH0,rH0,rW00 - stw rH0,0(rHP) - add rH1,rH1,rW14 - stw rH1,4(rHP) - add rH2,rH2,rW12 - stw rH2,8(rHP) - add rH3,rH3,rW10 - stw rH3,12(rHP) - NEXT_BLOCK - - bdnz ppc_md5_main - - FINALIZE - blr diff --git a/lib/crypto/powerpc/md5.h b/lib/crypto/powerpc/md5.h deleted file mode 100644 index 540b08e34d1d..000000000000 --- a/lib/crypto/powerpc/md5.h +++ /dev/null @@ -1,12 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * MD5 optimized for PowerPC - */ - -void ppc_md5_transform(u32 *state, const u8 *data, size_t nblocks); - -static void md5_blocks(struct md5_block_state *state, - const u8 *data, size_t nblocks) -{ - ppc_md5_transform(state->h, data, nblocks); -} diff --git a/lib/raid/xor/Makefile b/lib/raid/xor/Makefile index 4d633dfd5b90..e8ecec3c09f9 100644 --- a/lib/raid/xor/Makefile +++ b/lib/raid/xor/Makefile @@ -17,9 +17,10 @@ endif xor-$(CONFIG_ALPHA) += alpha/xor.o xor-$(CONFIG_ARM) += arm/xor.o ifeq ($(CONFIG_ARM),y) -xor-$(CONFIG_KERNEL_MODE_NEON) += arm/xor-neon.o arm/xor-neon-glue.o +xor-$(CONFIG_KERNEL_MODE_NEON) += xor-neon.o arm/xor-neon-glue.o endif -xor-$(CONFIG_ARM64) += arm64/xor-neon.o arm64/xor-neon-glue.o +xor-$(CONFIG_ARM64) += xor-neon.o arm64/xor-eor3.o \ + arm64/xor-neon-glue.o xor-$(CONFIG_CPU_HAS_LSX) += loongarch/xor_simd.o xor-$(CONFIG_CPU_HAS_LSX) += loongarch/xor_simd_glue.o xor-$(CONFIG_ALTIVEC) += powerpc/xor_vmx.o powerpc/xor_vmx_glue.o @@ -31,11 +32,11 @@ xor-$(CONFIG_X86_32) += x86/xor-avx.o x86/xor-sse.o x86/xor-mmx.o xor-$(CONFIG_X86_64) += x86/xor-avx.o x86/xor-sse.o obj-y += tests/ -CFLAGS_arm/xor-neon.o += $(CC_FLAGS_FPU) -CFLAGS_REMOVE_arm/xor-neon.o += $(CC_FLAGS_NO_FPU) +CFLAGS_xor-neon.o += $(CC_FLAGS_FPU) -I$(src)/$(SRCARCH) +CFLAGS_REMOVE_xor-neon.o += $(CC_FLAGS_NO_FPU) -CFLAGS_arm64/xor-neon.o += $(CC_FLAGS_FPU) -CFLAGS_REMOVE_arm64/xor-neon.o += $(CC_FLAGS_NO_FPU) +CFLAGS_arm64/xor-eor3.o += $(CC_FLAGS_FPU) +CFLAGS_REMOVE_arm64/xor-eor3.o += $(CC_FLAGS_NO_FPU) CFLAGS_powerpc/xor_vmx.o += -mhard-float -maltivec \ $(call cc-option,-mabi=altivec) \ diff --git a/lib/raid/xor/arm/xor-neon.c b/lib/raid/xor/arm/xor-neon.c deleted file mode 100644 index 23147e3a7904..000000000000 --- a/lib/raid/xor/arm/xor-neon.c +++ /dev/null @@ -1,26 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2013 Linaro Ltd - */ - -#include "xor_impl.h" -#include "xor_arch.h" - -#ifndef __ARM_NEON__ -#error You should compile this file with '-march=armv7-a -mfloat-abi=softfp -mfpu=neon' -#endif - -/* - * Pull in the reference implementations while instructing GCC (through - * -ftree-vectorize) to attempt to exploit implicit parallelism and emit - * NEON instructions. Clang does this by default at O2 so no pragma is - * needed. - */ -#ifdef CONFIG_CC_IS_GCC -#pragma GCC optimize "tree-vectorize" -#endif - -#define NO_TEMPLATE -#include "../xor-8regs.c" - -__DO_XOR_BLOCKS(neon_inner, xor_8regs_2, xor_8regs_3, xor_8regs_4, xor_8regs_5); diff --git a/lib/raid/xor/arm/xor-neon.h b/lib/raid/xor/arm/xor-neon.h new file mode 100644 index 000000000000..406e0356f05b --- /dev/null +++ b/lib/raid/xor/arm/xor-neon.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +extern struct xor_block_template xor_block_arm4regs; +extern struct xor_block_template xor_block_neon; + +void xor_gen_neon_inner(void *dest, void **srcs, unsigned int src_cnt, + unsigned int bytes); diff --git a/lib/raid/xor/arm/xor_arch.h b/lib/raid/xor/arm/xor_arch.h index 775ff835df65..f1ddb64fe62a 100644 --- a/lib/raid/xor/arm/xor_arch.h +++ b/lib/raid/xor/arm/xor_arch.h @@ -3,12 +3,7 @@ * Copyright (C) 2001 Russell King */ #include - -extern struct xor_block_template xor_block_arm4regs; -extern struct xor_block_template xor_block_neon; - -void xor_gen_neon_inner(void *dest, void **srcs, unsigned int src_cnt, - unsigned int bytes); +#include "xor-neon.h" static __always_inline void __init arch_xor_init(void) { diff --git a/lib/raid/xor/arm64/xor-eor3.c b/lib/raid/xor/arm64/xor-eor3.c new file mode 100644 index 000000000000..e44016c363f1 --- /dev/null +++ b/lib/raid/xor/arm64/xor-eor3.c @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include "xor_impl.h" +#include "xor_arch.h" +#include "xor-neon.h" + +extern void __xor_eor3_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2); + +static inline uint64x2_t eor3(uint64x2_t p, uint64x2_t q, uint64x2_t r) +{ + uint64x2_t res; + + asm(ARM64_ASM_PREAMBLE ".arch_extension sha3\n" + "eor3 %0.16b, %1.16b, %2.16b, %3.16b" + : "=w"(res) : "w"(p), "w"(q), "w"(r)); + return res; +} + +static void __xor_eor3_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) +{ + uint64_t *dp1 = (uint64_t *)p1; + uint64_t *dp2 = (uint64_t *)p2; + uint64_t *dp3 = (uint64_t *)p3; + + register uint64x2_t v0, v1, v2, v3; + long lines = bytes / (sizeof(uint64x2_t) * 4); + + do { + /* p1 ^= p2 ^ p3 */ + v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0), + vld1q_u64(dp3 + 0)); + v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2), + vld1q_u64(dp3 + 2)); + v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4), + vld1q_u64(dp3 + 4)); + v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6), + vld1q_u64(dp3 + 6)); + + /* store */ + vst1q_u64(dp1 + 0, v0); + vst1q_u64(dp1 + 2, v1); + vst1q_u64(dp1 + 4, v2); + vst1q_u64(dp1 + 6, v3); + + dp1 += 8; + dp2 += 8; + dp3 += 8; + } while (--lines > 0); +} + +static void __xor_eor3_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) +{ + uint64_t *dp1 = (uint64_t *)p1; + uint64_t *dp2 = (uint64_t *)p2; + uint64_t *dp3 = (uint64_t *)p3; + uint64_t *dp4 = (uint64_t *)p4; + + register uint64x2_t v0, v1, v2, v3; + long lines = bytes / (sizeof(uint64x2_t) * 4); + + do { + /* p1 ^= p2 ^ p3 */ + v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0), + vld1q_u64(dp3 + 0)); + v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2), + vld1q_u64(dp3 + 2)); + v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4), + vld1q_u64(dp3 + 4)); + v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6), + vld1q_u64(dp3 + 6)); + + /* p1 ^= p4 */ + v0 = veorq_u64(v0, vld1q_u64(dp4 + 0)); + v1 = veorq_u64(v1, vld1q_u64(dp4 + 2)); + v2 = veorq_u64(v2, vld1q_u64(dp4 + 4)); + v3 = veorq_u64(v3, vld1q_u64(dp4 + 6)); + + /* store */ + vst1q_u64(dp1 + 0, v0); + vst1q_u64(dp1 + 2, v1); + vst1q_u64(dp1 + 4, v2); + vst1q_u64(dp1 + 6, v3); + + dp1 += 8; + dp2 += 8; + dp3 += 8; + dp4 += 8; + } while (--lines > 0); +} + +static void __xor_eor3_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) +{ + uint64_t *dp1 = (uint64_t *)p1; + uint64_t *dp2 = (uint64_t *)p2; + uint64_t *dp3 = (uint64_t *)p3; + uint64_t *dp4 = (uint64_t *)p4; + uint64_t *dp5 = (uint64_t *)p5; + + register uint64x2_t v0, v1, v2, v3; + long lines = bytes / (sizeof(uint64x2_t) * 4); + + do { + /* p1 ^= p2 ^ p3 */ + v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0), + vld1q_u64(dp3 + 0)); + v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2), + vld1q_u64(dp3 + 2)); + v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4), + vld1q_u64(dp3 + 4)); + v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6), + vld1q_u64(dp3 + 6)); + + /* p1 ^= p4 ^ p5 */ + v0 = eor3(v0, vld1q_u64(dp4 + 0), vld1q_u64(dp5 + 0)); + v1 = eor3(v1, vld1q_u64(dp4 + 2), vld1q_u64(dp5 + 2)); + v2 = eor3(v2, vld1q_u64(dp4 + 4), vld1q_u64(dp5 + 4)); + v3 = eor3(v3, vld1q_u64(dp4 + 6), vld1q_u64(dp5 + 6)); + + /* store */ + vst1q_u64(dp1 + 0, v0); + vst1q_u64(dp1 + 2, v1); + vst1q_u64(dp1 + 4, v2); + vst1q_u64(dp1 + 6, v3); + + dp1 += 8; + dp2 += 8; + dp3 += 8; + dp4 += 8; + dp5 += 8; + } while (--lines > 0); +} + +__DO_XOR_BLOCKS(eor3_inner, __xor_eor3_2, __xor_eor3_3, __xor_eor3_4, + __xor_eor3_5); diff --git a/lib/raid/xor/xor-8regs.c b/lib/raid/xor/xor-8regs.c index 1edaed8acffe..46b3c8bdc27f 100644 --- a/lib/raid/xor/xor-8regs.c +++ b/lib/raid/xor/xor-8regs.c @@ -93,11 +93,9 @@ xor_8regs_5(unsigned long bytes, unsigned long * __restrict p1, } while (--lines > 0); } -#ifndef NO_TEMPLATE DO_XOR_BLOCKS(8regs, xor_8regs_2, xor_8regs_3, xor_8regs_4, xor_8regs_5); struct xor_block_template xor_block_8regs = { .name = "8regs", .xor_gen = xor_gen_8regs, }; -#endif /* NO_TEMPLATE */ diff --git a/lib/raid/xor/arm64/xor-neon.c b/lib/raid/xor/xor-neon.c similarity index 56% rename from lib/raid/xor/arm64/xor-neon.c rename to lib/raid/xor/xor-neon.c index 97ef3cb92496..c7c3cf634e23 100644 --- a/lib/raid/xor/arm64/xor-neon.c +++ b/lib/raid/xor/xor-neon.c @@ -4,12 +4,11 @@ * Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd. */ -#include -#include #include "xor_impl.h" -#include "xor_arch.h" #include "xor-neon.h" +#include + static void __xor_neon_2(unsigned long bytes, unsigned long * __restrict p1, const unsigned long * __restrict p2) { @@ -175,138 +174,6 @@ static void __xor_neon_5(unsigned long bytes, unsigned long * __restrict p1, __DO_XOR_BLOCKS(neon_inner, __xor_neon_2, __xor_neon_3, __xor_neon_4, __xor_neon_5); -static inline uint64x2_t eor3(uint64x2_t p, uint64x2_t q, uint64x2_t r) -{ - uint64x2_t res; - - asm(ARM64_ASM_PREAMBLE ".arch_extension sha3\n" - "eor3 %0.16b, %1.16b, %2.16b, %3.16b" - : "=w"(res) : "w"(p), "w"(q), "w"(r)); - return res; -} - -static void __xor_eor3_3(unsigned long bytes, unsigned long * __restrict p1, - const unsigned long * __restrict p2, - const unsigned long * __restrict p3) -{ - uint64_t *dp1 = (uint64_t *)p1; - uint64_t *dp2 = (uint64_t *)p2; - uint64_t *dp3 = (uint64_t *)p3; - - register uint64x2_t v0, v1, v2, v3; - long lines = bytes / (sizeof(uint64x2_t) * 4); - - do { - /* p1 ^= p2 ^ p3 */ - v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0), - vld1q_u64(dp3 + 0)); - v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2), - vld1q_u64(dp3 + 2)); - v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4), - vld1q_u64(dp3 + 4)); - v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6), - vld1q_u64(dp3 + 6)); - - /* store */ - vst1q_u64(dp1 + 0, v0); - vst1q_u64(dp1 + 2, v1); - vst1q_u64(dp1 + 4, v2); - vst1q_u64(dp1 + 6, v3); - - dp1 += 8; - dp2 += 8; - dp3 += 8; - } while (--lines > 0); -} - -static void __xor_eor3_4(unsigned long bytes, unsigned long * __restrict p1, - const unsigned long * __restrict p2, - const unsigned long * __restrict p3, - const unsigned long * __restrict p4) -{ - uint64_t *dp1 = (uint64_t *)p1; - uint64_t *dp2 = (uint64_t *)p2; - uint64_t *dp3 = (uint64_t *)p3; - uint64_t *dp4 = (uint64_t *)p4; - - register uint64x2_t v0, v1, v2, v3; - long lines = bytes / (sizeof(uint64x2_t) * 4); - - do { - /* p1 ^= p2 ^ p3 */ - v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0), - vld1q_u64(dp3 + 0)); - v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2), - vld1q_u64(dp3 + 2)); - v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4), - vld1q_u64(dp3 + 4)); - v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6), - vld1q_u64(dp3 + 6)); - - /* p1 ^= p4 */ - v0 = veorq_u64(v0, vld1q_u64(dp4 + 0)); - v1 = veorq_u64(v1, vld1q_u64(dp4 + 2)); - v2 = veorq_u64(v2, vld1q_u64(dp4 + 4)); - v3 = veorq_u64(v3, vld1q_u64(dp4 + 6)); - - /* store */ - vst1q_u64(dp1 + 0, v0); - vst1q_u64(dp1 + 2, v1); - vst1q_u64(dp1 + 4, v2); - vst1q_u64(dp1 + 6, v3); - - dp1 += 8; - dp2 += 8; - dp3 += 8; - dp4 += 8; - } while (--lines > 0); -} - -static void __xor_eor3_5(unsigned long bytes, unsigned long * __restrict p1, - const unsigned long * __restrict p2, - const unsigned long * __restrict p3, - const unsigned long * __restrict p4, - const unsigned long * __restrict p5) -{ - uint64_t *dp1 = (uint64_t *)p1; - uint64_t *dp2 = (uint64_t *)p2; - uint64_t *dp3 = (uint64_t *)p3; - uint64_t *dp4 = (uint64_t *)p4; - uint64_t *dp5 = (uint64_t *)p5; - - register uint64x2_t v0, v1, v2, v3; - long lines = bytes / (sizeof(uint64x2_t) * 4); - - do { - /* p1 ^= p2 ^ p3 */ - v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0), - vld1q_u64(dp3 + 0)); - v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2), - vld1q_u64(dp3 + 2)); - v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4), - vld1q_u64(dp3 + 4)); - v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6), - vld1q_u64(dp3 + 6)); - - /* p1 ^= p4 ^ p5 */ - v0 = eor3(v0, vld1q_u64(dp4 + 0), vld1q_u64(dp5 + 0)); - v1 = eor3(v1, vld1q_u64(dp4 + 2), vld1q_u64(dp5 + 2)); - v2 = eor3(v2, vld1q_u64(dp4 + 4), vld1q_u64(dp5 + 4)); - v3 = eor3(v3, vld1q_u64(dp4 + 6), vld1q_u64(dp5 + 6)); - - /* store */ - vst1q_u64(dp1 + 0, v0); - vst1q_u64(dp1 + 2, v1); - vst1q_u64(dp1 + 4, v2); - vst1q_u64(dp1 + 6, v3); - - dp1 += 8; - dp2 += 8; - dp3 += 8; - dp4 += 8; - dp5 += 8; - } while (--lines > 0); -} - -__DO_XOR_BLOCKS(eor3_inner, __xor_neon_2, __xor_eor3_3, __xor_eor3_4, - __xor_eor3_5); +#ifdef CONFIG_ARM64 +extern typeof(__xor_neon_2) __xor_eor3_2 __alias(__xor_neon_2); +#endif diff --git a/lib/string_helpers.c b/lib/string_helpers.c index 169eaf583494..98d6ed0eaab7 100644 --- a/lib/string_helpers.c +++ b/lib/string_helpers.c @@ -752,7 +752,7 @@ EXPORT_SYMBOL_GPL(kstrdup_and_replace); * kasprintf_strarray - allocate and fill array of sequential strings * @gfp: flags for the slab allocator * @prefix: prefix to be used - * @n: amount of lines to be allocated and filled + * @n: number of strings to be allocated and filled * * Allocates and fills @n strings using pattern "%s-%zu", where prefix * is provided by caller. The caller is responsible to free them with @@ -765,7 +765,7 @@ char **kasprintf_strarray(gfp_t gfp, const char *prefix, size_t n) char **names; size_t i; - names = kcalloc(n + 1, sizeof(char *), gfp); + names = kcalloc(n, sizeof(char *), gfp); if (!names) return NULL; @@ -805,7 +805,7 @@ void kfree_strarray(char **array, size_t n) EXPORT_SYMBOL_GPL(kfree_strarray); struct strarray { - char **array; + char **array __counted_by_ptr(n); size_t n; }; @@ -824,13 +824,13 @@ char **devm_kasprintf_strarray(struct device *dev, const char *prefix, size_t n) if (!ptr) return ERR_PTR(-ENOMEM); + ptr->n = n; ptr->array = kasprintf_strarray(GFP_KERNEL, prefix, n); if (!ptr->array) { devres_free(ptr); return ERR_PTR(-ENOMEM); } - ptr->n = n; devres_add(dev, ptr); return ptr->array; diff --git a/tools/testing/selftests/lkdtm/tests.txt b/tools/testing/selftests/lkdtm/tests.txt index 3245032db34d..d8180bbe31e8 100644 --- a/tools/testing/selftests/lkdtm/tests.txt +++ b/tools/testing/selftests/lkdtm/tests.txt @@ -86,3 +86,4 @@ FORTIFY_STR_MEMBER detected buffer overflow FORTIFY_MEM_OBJECT detected buffer overflow FORTIFY_MEM_MEMBER detected field-spanning write PPC_SLB_MULTIHIT Recovered +#PPC_RADIX_TLBIEL Triggers unrecoverable MCE