From d16a5f852025be546b6e4ceef15899db3490f4d7 Mon Sep 17 00:00:00 2001 From: Kemeng Shi Date: Mon, 15 Jul 2024 21:05:31 +0800 Subject: [PATCH 001/302] quota: avoid missing put_quota_format when DQUOT_SUSPENDED is passed Avoid missing put_quota_format when DQUOT_SUSPENDED is passed to dquot_load_quota_sb. Link: https://patch.msgid.link/20240715130534.2112678-2-shikemeng@huaweicloud.com Signed-off-by: Kemeng Shi Fixes: d44c57663723 ("quota: Remove BUG_ON in dquot_load_quota_sb()") Reviewed-by: Joseph Qi Signed-off-by: Jan Kara --- fs/quota/dquot.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/quota/dquot.c b/fs/quota/dquot.c index a2b256dac36e..104927804bec 100644 --- a/fs/quota/dquot.c +++ b/fs/quota/dquot.c @@ -2406,7 +2406,7 @@ static int vfs_setup_quota_inode(struct inode *inode, int type) int dquot_load_quota_sb(struct super_block *sb, int type, int format_id, unsigned int flags) { - struct quota_format_type *fmt = find_quota_format(format_id); + struct quota_format_type *fmt; struct quota_info *dqopt = sb_dqopt(sb); int error; @@ -2416,6 +2416,7 @@ int dquot_load_quota_sb(struct super_block *sb, int type, int format_id, if (WARN_ON_ONCE(flags & DQUOT_SUSPENDED)) return -EINVAL; + fmt = find_quota_format(format_id); if (!fmt) return -ESRCH; if (!sb->dq_op || !sb->s_qcop || From a838e5dca63d1dc701e63b2b1176943c57485c45 Mon Sep 17 00:00:00 2001 From: Kemeng Shi Date: Mon, 15 Jul 2024 21:05:32 +0800 Subject: [PATCH 002/302] quota: remove unneeded return value of register_quota_format The register_quota_format always returns 0, simply remove unneeded return value. Link: https://patch.msgid.link/20240715130534.2112678-3-shikemeng@huaweicloud.com Signed-off-by: Kemeng Shi Reviewed-by: Joseph Qi Signed-off-by: Jan Kara --- fs/ocfs2/super.c | 6 ++---- fs/quota/dquot.c | 3 +-- fs/quota/quota_v1.c | 3 ++- fs/quota/quota_v2.c | 9 +++------ include/linux/quota.h | 2 +- mm/shmem.c | 7 +------ 6 files changed, 10 insertions(+), 20 deletions(-) diff --git a/fs/ocfs2/super.c b/fs/ocfs2/super.c index afee70125ae3..73caa8914ebe 100644 --- a/fs/ocfs2/super.c +++ b/fs/ocfs2/super.c @@ -1571,15 +1571,13 @@ static int __init ocfs2_init(void) ocfs2_set_locking_protocol(); - status = register_quota_format(&ocfs2_quota_format); - if (status < 0) - goto out3; + register_quota_format(&ocfs2_quota_format); + status = register_filesystem(&ocfs2_fs_type); if (!status) return 0; unregister_quota_format(&ocfs2_quota_format); -out3: debugfs_remove(ocfs2_debugfs_root); ocfs2_free_mem_caches(); out2: diff --git a/fs/quota/dquot.c b/fs/quota/dquot.c index 104927804bec..1ddd1e9d1eeb 100644 --- a/fs/quota/dquot.c +++ b/fs/quota/dquot.c @@ -163,13 +163,12 @@ static struct quota_module_name module_names[] = INIT_QUOTA_MODULE_NAMES; /* SLAB cache for dquot structures */ static struct kmem_cache *dquot_cachep; -int register_quota_format(struct quota_format_type *fmt) +void register_quota_format(struct quota_format_type *fmt) { spin_lock(&dq_list_lock); fmt->qf_next = quota_formats; quota_formats = fmt; spin_unlock(&dq_list_lock); - return 0; } EXPORT_SYMBOL(register_quota_format); diff --git a/fs/quota/quota_v1.c b/fs/quota/quota_v1.c index 3f3e8acc05db..6f7f0b4afba9 100644 --- a/fs/quota/quota_v1.c +++ b/fs/quota/quota_v1.c @@ -235,7 +235,8 @@ static struct quota_format_type v1_quota_format = { static int __init init_v1_quota_format(void) { - return register_quota_format(&v1_quota_format); + register_quota_format(&v1_quota_format); + return 0; } static void __exit exit_v1_quota_format(void) diff --git a/fs/quota/quota_v2.c b/fs/quota/quota_v2.c index c48c233f3bef..1fda93dcbc1b 100644 --- a/fs/quota/quota_v2.c +++ b/fs/quota/quota_v2.c @@ -440,12 +440,9 @@ static struct quota_format_type v2r1_quota_format = { static int __init init_v2_quota_format(void) { - int ret; - - ret = register_quota_format(&v2r0_quota_format); - if (ret) - return ret; - return register_quota_format(&v2r1_quota_format); + register_quota_format(&v2r0_quota_format); + register_quota_format(&v2r1_quota_format); + return 0; } static void __exit exit_v2_quota_format(void) diff --git a/include/linux/quota.h b/include/linux/quota.h index 07071e64abf3..89a0d83ddad0 100644 --- a/include/linux/quota.h +++ b/include/linux/quota.h @@ -526,7 +526,7 @@ struct quota_info { const struct quota_format_ops *ops[MAXQUOTAS]; /* Operations for each type */ }; -int register_quota_format(struct quota_format_type *fmt); +void register_quota_format(struct quota_format_type *fmt); void unregister_quota_format(struct quota_format_type *fmt); struct quota_module_name { diff --git a/mm/shmem.c b/mm/shmem.c index 2faa9daaf54b..3b922d44e12c 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -4818,11 +4818,7 @@ void __init shmem_init(void) shmem_init_inodecache(); #ifdef CONFIG_TMPFS_QUOTA - error = register_quota_format(&shmem_quota_format); - if (error < 0) { - pr_err("Could not register quota format\n"); - goto out3; - } + register_quota_format(&shmem_quota_format); #endif error = register_filesystem(&shmem_fs_type); @@ -4857,7 +4853,6 @@ out1: out2: #ifdef CONFIG_TMPFS_QUOTA unregister_quota_format(&shmem_quota_format); -out3: #endif shmem_destroy_inodecache(); shm_mnt = ERR_PTR(error); From f439111b489167dce14ade4e848be8a1b23283ff Mon Sep 17 00:00:00 2001 From: Kemeng Shi Date: Mon, 15 Jul 2024 21:05:33 +0800 Subject: [PATCH 003/302] quota: remove redundant return at end of void function Function dquot_claim_space_nodirty nad dquot_reclaim_space_nodirty have no return value, just remove redundant return at end of the functions. Link: https://patch.msgid.link/20240715130534.2112678-4-shikemeng@huaweicloud.com Signed-off-by: Kemeng Shi Signed-off-by: Jan Kara --- fs/quota/dquot.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/fs/quota/dquot.c b/fs/quota/dquot.c index 1ddd1e9d1eeb..156bdd570ef6 100644 --- a/fs/quota/dquot.c +++ b/fs/quota/dquot.c @@ -1830,7 +1830,6 @@ void dquot_claim_space_nodirty(struct inode *inode, qsize_t number) spin_unlock(&inode->i_lock); mark_all_dquot_dirty(dquots); srcu_read_unlock(&dquot_srcu, index); - return; } EXPORT_SYMBOL(dquot_claim_space_nodirty); @@ -1872,7 +1871,6 @@ void dquot_reclaim_space_nodirty(struct inode *inode, qsize_t number) spin_unlock(&inode->i_lock); mark_all_dquot_dirty(dquots); srcu_read_unlock(&dquot_srcu, index); - return; } EXPORT_SYMBOL(dquot_reclaim_space_nodirty); From edb46ddbc55e53eef3d6e575087baa315598d253 Mon Sep 17 00:00:00 2001 From: Kemeng Shi Date: Mon, 15 Jul 2024 21:05:34 +0800 Subject: [PATCH 004/302] quota: remove unnecessary error code translation in dquot_quota_enable Simply set error code to -EEXIST when quota limit is already enabled in dquot_quota_enable to remove unnecessary error code translation. Link: https://patch.msgid.link/20240715130534.2112678-5-shikemeng@huaweicloud.com Signed-off-by: Kemeng Shi Signed-off-by: Jan Kara --- fs/quota/dquot.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/fs/quota/dquot.c b/fs/quota/dquot.c index 156bdd570ef6..30496367d581 100644 --- a/fs/quota/dquot.c +++ b/fs/quota/dquot.c @@ -2594,7 +2594,8 @@ static int dquot_quota_enable(struct super_block *sb, unsigned int flags) goto out_err; } if (sb_has_quota_limits_enabled(sb, type)) { - ret = -EBUSY; + /* compatible with XFS */ + ret = -EEXIST; goto out_err; } spin_lock(&dq_state_lock); @@ -2608,9 +2609,6 @@ out_err: if (flags & qtype_enforce_flag(type)) dquot_disable(sb, type, DQUOT_LIMITS_ENABLED); } - /* Error code translation for better compatibility with XFS */ - if (ret == -EBUSY) - ret = -EEXIST; return ret; } From 5297bba507dc54045e6efeb9955c1271ca9aafe1 Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Tue, 23 Jul 2024 15:27:01 +0200 Subject: [PATCH 005/302] genirq/msi: Silence 'set affinity failed' warning Various PCI controllers that mux MSIs onto a single IRQ line produce these "IRQ%d: set affinity failed" warnings when entering suspend. This has been discussed before [1] [2] and an example test case is included at the end of this commit message. Controller drivers that create MSI IRQ domain with MSI_FLAG_USE_DEF_CHIP_OPS and do not override the .irq_set_affinity() irqchip callback get assigned the default msi_domain_set_affinity() callback. That is not desired on controllers where it is not possible to set affinity of each MSI IRQ line to a specific CPU core due to hardware limitation. Introduce flag MSI_FLAG_NO_AFFINITY, which keeps .irq_set_affinity() unset if the controller driver did not assign it. This way, migrate_one_irq() can exit right away, without printing the warning. The .irq_set_affinity() implementations which only return -EINVAL can be removed from multiple controller drivers. $ grep 25 /proc/interrupts 25: 0 0 0 0 0 0 0 0 PCIe MSI 0 Edge PCIe PME $ echo core > /sys/power/pm_test ; echo mem > /sys/power/state ... Disabling non-boot CPUs ... IRQ25: set affinity failed(-22). <---------- This is being silenced here psci: CPU7 killed (polled 4 ms) ... [1] https://lore.kernel.org/all/d4a6eea3c5e33a3a4056885419df95a7@kernel.org/ [2] https://lore.kernel.org/all/5f4947b18bf381615a37aa81c2242477@kernel.org/ Link: https://lore.kernel.org/r/20240723132958.41320-2-marek.vasut+renesas@mailbox.org Signed-off-by: Marek Vasut [bhelgaas: commit log] Signed-off-by: Bjorn Helgaas Reviewed-by: Damien Le Moal Reviewed-by: Manivannan Sadhasivam Acked-by: Thomas Gleixner --- include/linux/msi.h | 2 ++ kernel/irq/msi.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/include/linux/msi.h b/include/linux/msi.h index 944979763825..b10093c4d00e 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -554,6 +554,8 @@ enum { MSI_FLAG_MSIX_CONTIGUOUS = (1 << 19), /* PCI/MSI-X vectors can be dynamically allocated/freed post MSI-X enable */ MSI_FLAG_PCI_MSIX_ALLOC_DYN = (1 << 20), + /* PCI MSIs cannot be steered separately to CPU cores */ + MSI_FLAG_NO_AFFINITY = (1 << 21), }; /** diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c index 5fa0547ece0c..ca6e2ae6d6fc 100644 --- a/kernel/irq/msi.c +++ b/kernel/irq/msi.c @@ -832,7 +832,7 @@ static void msi_domain_update_chip_ops(struct msi_domain_info *info) struct irq_chip *chip = info->chip; BUG_ON(!chip || !chip->irq_mask || !chip->irq_unmask); - if (!chip->irq_set_affinity) + if (!chip->irq_set_affinity && !(info->flags & MSI_FLAG_NO_AFFINITY)) chip->irq_set_affinity = msi_domain_set_affinity; } From 4dff9c32ae066896a77c23628fb221c4cb9f5111 Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Tue, 23 Jul 2024 15:27:02 +0200 Subject: [PATCH 006/302] PCI: aardvark: Silence 'set affinity failed' warning Use MSI_FLAG_NO_AFFINITY, which keeps .irq_set_affinity() unset and allows migrate_one_irq() to exit right away, without warnings like this: IRQ...: set affinity failed(-22) Remove the .irq_set_affinity() implementation that is no longer needed. Link: https://lore.kernel.org/r/20240723132958.41320-3-marek.vasut+renesas@mailbox.org Signed-off-by: Marek Vasut [bhelgaas: commit log] Signed-off-by: Bjorn Helgaas Reviewed-by: Manivannan Sadhasivam Acked-by: Thomas Gleixner --- drivers/pci/controller/pci-aardvark.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/drivers/pci/controller/pci-aardvark.c b/drivers/pci/controller/pci-aardvark.c index 8b3e1a079cf3..ea9b72e0269d 100644 --- a/drivers/pci/controller/pci-aardvark.c +++ b/drivers/pci/controller/pci-aardvark.c @@ -1304,12 +1304,6 @@ static void advk_msi_irq_compose_msi_msg(struct irq_data *data, msg->data = data->hwirq; } -static int advk_msi_set_affinity(struct irq_data *irq_data, - const struct cpumask *mask, bool force) -{ - return -EINVAL; -} - static void advk_msi_irq_mask(struct irq_data *d) { struct advk_pcie *pcie = d->domain->host_data; @@ -1353,7 +1347,6 @@ static void advk_msi_top_irq_unmask(struct irq_data *d) static struct irq_chip advk_msi_bottom_irq_chip = { .name = "MSI", .irq_compose_msi_msg = advk_msi_irq_compose_msi_msg, - .irq_set_affinity = advk_msi_set_affinity, .irq_mask = advk_msi_irq_mask, .irq_unmask = advk_msi_irq_unmask, }; @@ -1451,7 +1444,8 @@ static struct irq_chip advk_msi_irq_chip = { static struct msi_domain_info advk_msi_domain_info = { .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | - MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX, + MSI_FLAG_NO_AFFINITY | MSI_FLAG_MULTI_PCI_MSI | + MSI_FLAG_PCI_MSIX, .chip = &advk_msi_irq_chip, }; From 114ca2969f1e4d4f3b6920a4824f11b0a9cf708a Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Tue, 23 Jul 2024 15:27:03 +0200 Subject: [PATCH 007/302] PCI: altera-msi: Silence 'set affinity failed' warning Use MSI_FLAG_NO_AFFINITY, which keeps .irq_set_affinity() unset and allows migrate_one_irq() to exit right away, without warnings like this: IRQ...: set affinity failed(-22) Remove the .irq_set_affinity() implementation that is no longer needed. Link: https://lore.kernel.org/r/20240723132958.41320-4-marek.vasut+renesas@mailbox.org Signed-off-by: Marek Vasut [bhelgaas: commit log] Signed-off-by: Bjorn Helgaas Reviewed-by: Manivannan Sadhasivam Acked-by: Thomas Gleixner --- drivers/pci/controller/pcie-altera-msi.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/drivers/pci/controller/pcie-altera-msi.c b/drivers/pci/controller/pcie-altera-msi.c index 16336a525c16..e36a6e158d23 100644 --- a/drivers/pci/controller/pcie-altera-msi.c +++ b/drivers/pci/controller/pcie-altera-msi.c @@ -81,8 +81,8 @@ static struct irq_chip altera_msi_irq_chip = { }; static struct msi_domain_info altera_msi_domain_info = { - .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | - MSI_FLAG_PCI_MSIX), + .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_NO_AFFINITY | MSI_FLAG_PCI_MSIX, .chip = &altera_msi_irq_chip, }; @@ -99,16 +99,9 @@ static void altera_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) (int)data->hwirq, msg->address_hi, msg->address_lo); } -static int altera_msi_set_affinity(struct irq_data *irq_data, - const struct cpumask *mask, bool force) -{ - return -EINVAL; -} - static struct irq_chip altera_msi_bottom_irq_chip = { .name = "Altera MSI", .irq_compose_msi_msg = altera_compose_msi_msg, - .irq_set_affinity = altera_msi_set_affinity, }; static int altera_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, From c7b10edde0f25edd74a84f51a195bdf3a634d473 Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Tue, 23 Jul 2024 15:27:04 +0200 Subject: [PATCH 008/302] PCI: brcmstb: Silence 'set affinity failed' warning Use MSI_FLAG_NO_AFFINITY, which keeps .irq_set_affinity() unset and allows migrate_one_irq() to exit right away, without warnings like this: IRQ...: set affinity failed(-22) Remove the .irq_set_affinity() implementation that is no longer needed. Link: https://lore.kernel.org/r/20240723132958.41320-5-marek.vasut+renesas@mailbox.org Signed-off-by: Marek Vasut [bhelgaas: commit log] Signed-off-by: Bjorn Helgaas Reviewed-by: Florian Fainelli Reviewed-by: Manivannan Sadhasivam Acked-by: Thomas Gleixner --- drivers/pci/controller/pcie-brcmstb.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/controller/pcie-brcmstb.c index c08683febdd4..b9de3acec375 100644 --- a/drivers/pci/controller/pcie-brcmstb.c +++ b/drivers/pci/controller/pcie-brcmstb.c @@ -445,8 +445,8 @@ static struct irq_chip brcm_msi_irq_chip = { }; static struct msi_domain_info brcm_msi_domain_info = { - .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | - MSI_FLAG_MULTI_PCI_MSI), + .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_NO_AFFINITY | MSI_FLAG_MULTI_PCI_MSI, .chip = &brcm_msi_irq_chip, }; @@ -484,12 +484,6 @@ static void brcm_msi_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) msg->data = (0xffff & PCIE_MISC_MSI_DATA_CONFIG_VAL_32) | data->hwirq; } -static int brcm_msi_set_affinity(struct irq_data *irq_data, - const struct cpumask *mask, bool force) -{ - return -EINVAL; -} - static void brcm_msi_ack_irq(struct irq_data *data) { struct brcm_msi *msi = irq_data_get_irq_chip_data(data); @@ -502,7 +496,6 @@ static void brcm_msi_ack_irq(struct irq_data *data) static struct irq_chip brcm_msi_bottom_irq_chip = { .name = "BRCM STB MSI", .irq_compose_msi_msg = brcm_msi_compose_msi_msg, - .irq_set_affinity = brcm_msi_set_affinity, .irq_ack = brcm_msi_ack_irq, }; From 363d53acbfd3ec0454349d1e312f600d3fa33285 Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Tue, 23 Jul 2024 15:27:05 +0200 Subject: [PATCH 009/302] PCI: dwc: Silence 'set affinity failed' warning Use MSI_FLAG_NO_AFFINITY, which keeps .irq_set_affinity() unset and allows migrate_one_irq() to exit right away, without warnings like this: IRQ...: set affinity failed(-22) Remove the .irq_set_affinity() implementation that is no longer needed. Link: https://lore.kernel.org/r/20240723132958.41320-6-marek.vasut+renesas@mailbox.org Signed-off-by: Marek Vasut [bhelgaas: commit log] Signed-off-by: Bjorn Helgaas Reviewed-by: Damien Le Moal Reviewed-by: Manivannan Sadhasivam Acked-by: Thomas Gleixner --- drivers/pci/controller/dwc/pci-keystone.c | 7 ------- drivers/pci/controller/dwc/pcie-designware-host.c | 12 +++--------- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/drivers/pci/controller/dwc/pci-keystone.c b/drivers/pci/controller/dwc/pci-keystone.c index 52c6420ae200..ce9d9e0a5260 100644 --- a/drivers/pci/controller/dwc/pci-keystone.c +++ b/drivers/pci/controller/dwc/pci-keystone.c @@ -189,12 +189,6 @@ static void ks_pcie_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) (int)data->hwirq, msg->address_hi, msg->address_lo); } -static int ks_pcie_msi_set_affinity(struct irq_data *irq_data, - const struct cpumask *mask, bool force) -{ - return -EINVAL; -} - static void ks_pcie_msi_mask(struct irq_data *data) { struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(data); @@ -247,7 +241,6 @@ static struct irq_chip ks_pcie_msi_irq_chip = { .name = "KEYSTONE-PCI-MSI", .irq_ack = ks_pcie_msi_irq_ack, .irq_compose_msi_msg = ks_pcie_compose_msi_msg, - .irq_set_affinity = ks_pcie_msi_set_affinity, .irq_mask = ks_pcie_msi_mask, .irq_unmask = ks_pcie_msi_unmask, }; diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c index a0822d5371bc..3e41865c7290 100644 --- a/drivers/pci/controller/dwc/pcie-designware-host.c +++ b/drivers/pci/controller/dwc/pcie-designware-host.c @@ -48,8 +48,9 @@ static struct irq_chip dw_pcie_msi_irq_chip = { }; static struct msi_domain_info dw_pcie_msi_domain_info = { - .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | - MSI_FLAG_PCI_MSIX | MSI_FLAG_MULTI_PCI_MSI), + .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_NO_AFFINITY | MSI_FLAG_PCI_MSIX | + MSI_FLAG_MULTI_PCI_MSI, .chip = &dw_pcie_msi_irq_chip, }; @@ -116,12 +117,6 @@ static void dw_pci_setup_msi_msg(struct irq_data *d, struct msi_msg *msg) (int)d->hwirq, msg->address_hi, msg->address_lo); } -static int dw_pci_msi_set_affinity(struct irq_data *d, - const struct cpumask *mask, bool force) -{ - return -EINVAL; -} - static void dw_pci_bottom_mask(struct irq_data *d) { struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d); @@ -177,7 +172,6 @@ static struct irq_chip dw_pci_msi_bottom_irq_chip = { .name = "DWPCI-MSI", .irq_ack = dw_pci_bottom_ack, .irq_compose_msi_msg = dw_pci_setup_msi_msg, - .irq_set_affinity = dw_pci_msi_set_affinity, .irq_mask = dw_pci_bottom_mask, .irq_unmask = dw_pci_bottom_unmask, }; From 425c07563f4cefdb4d521d1f1cf5990004a388fa Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Tue, 23 Jul 2024 15:27:06 +0200 Subject: [PATCH 010/302] PCI: mediatek-gen3: Silence 'set affinity failed' warning Use MSI_FLAG_NO_AFFINITY, which keeps .irq_set_affinity() unset and allows migrate_one_irq() to exit right away, without warnings like this: IRQ...: set affinity failed(-22) Remove the .irq_set_affinity() implementation that is no longer needed. Link: https://lore.kernel.org/r/20240723132958.41320-7-marek.vasut+renesas@mailbox.org Signed-off-by: Marek Vasut [bhelgaas: commit log] Signed-off-by: Bjorn Helgaas Reviewed-by: Manivannan Sadhasivam Acked-by: Jianjun Wang Acked-by: Thomas Gleixner --- drivers/pci/controller/pcie-mediatek-gen3.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/drivers/pci/controller/pcie-mediatek-gen3.c b/drivers/pci/controller/pcie-mediatek-gen3.c index b7e8e24f6a40..392b6ef3473f 100644 --- a/drivers/pci/controller/pcie-mediatek-gen3.c +++ b/drivers/pci/controller/pcie-mediatek-gen3.c @@ -424,12 +424,6 @@ static int mtk_pcie_startup_port(struct mtk_gen3_pcie *pcie) return 0; } -static int mtk_pcie_set_affinity(struct irq_data *data, - const struct cpumask *mask, bool force) -{ - return -EINVAL; -} - static void mtk_pcie_msi_irq_mask(struct irq_data *data) { pci_msi_mask_irq(data); @@ -450,8 +444,9 @@ static struct irq_chip mtk_msi_irq_chip = { }; static struct msi_domain_info mtk_msi_domain_info = { - .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | - MSI_FLAG_PCI_MSIX | MSI_FLAG_MULTI_PCI_MSI), + .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_NO_AFFINITY | MSI_FLAG_PCI_MSIX | + MSI_FLAG_MULTI_PCI_MSI, .chip = &mtk_msi_irq_chip, }; @@ -517,7 +512,6 @@ static struct irq_chip mtk_msi_bottom_irq_chip = { .irq_mask = mtk_msi_bottom_irq_mask, .irq_unmask = mtk_msi_bottom_irq_unmask, .irq_compose_msi_msg = mtk_compose_msi_msg, - .irq_set_affinity = mtk_pcie_set_affinity, .name = "MSI", }; @@ -618,7 +612,6 @@ static struct irq_chip mtk_intx_irq_chip = { .irq_mask = mtk_intx_mask, .irq_unmask = mtk_intx_unmask, .irq_eoi = mtk_intx_eoi, - .irq_set_affinity = mtk_pcie_set_affinity, .name = "INTx", }; From 63e4794cde79c2724f9c02abe8a46be70a5b90df Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Tue, 23 Jul 2024 15:27:07 +0200 Subject: [PATCH 011/302] PCI: mediatek: Silence 'set affinity failed' warning Use MSI_FLAG_NO_AFFINITY, which keeps .irq_set_affinity() unset and allows migrate_one_irq() to exit right away, without warnings like this: IRQ...: set affinity failed(-22) Remove the .irq_set_affinity() implementation that is no longer needed. Link: https://lore.kernel.org/r/20240723132958.41320-8-marek.vasut+renesas@mailbox.org Signed-off-by: Marek Vasut [bhelgaas: commit log] Signed-off-by: Bjorn Helgaas Reviewed-by: Manivannan Sadhasivam Acked-by: Jianjun Wang Acked-by: Thomas Gleixner --- drivers/pci/controller/pcie-mediatek.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/drivers/pci/controller/pcie-mediatek.c b/drivers/pci/controller/pcie-mediatek.c index 7fc0d7709b7f..9be3cebd862e 100644 --- a/drivers/pci/controller/pcie-mediatek.c +++ b/drivers/pci/controller/pcie-mediatek.c @@ -407,12 +407,6 @@ static void mtk_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) (int)data->hwirq, msg->address_hi, msg->address_lo); } -static int mtk_msi_set_affinity(struct irq_data *irq_data, - const struct cpumask *mask, bool force) -{ - return -EINVAL; -} - static void mtk_msi_ack_irq(struct irq_data *data) { struct mtk_pcie_port *port = irq_data_get_irq_chip_data(data); @@ -424,7 +418,6 @@ static void mtk_msi_ack_irq(struct irq_data *data) static struct irq_chip mtk_msi_bottom_irq_chip = { .name = "MTK MSI", .irq_compose_msi_msg = mtk_compose_msi_msg, - .irq_set_affinity = mtk_msi_set_affinity, .irq_ack = mtk_msi_ack_irq, }; @@ -486,8 +479,8 @@ static struct irq_chip mtk_msi_irq_chip = { }; static struct msi_domain_info mtk_msi_domain_info = { - .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | - MSI_FLAG_PCI_MSIX), + .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_NO_AFFINITY | MSI_FLAG_PCI_MSIX, .chip = &mtk_msi_irq_chip, }; From 2a1297ead0694c54e6997fe89f71b10e144d5540 Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Tue, 23 Jul 2024 15:27:08 +0200 Subject: [PATCH 012/302] PCI: mobiveil: Silence 'set affinity failed' warning Use MSI_FLAG_NO_AFFINITY, which keeps .irq_set_affinity() unset and allows migrate_one_irq() to exit right away, without warnings like this: IRQ...: set affinity failed(-22) Remove the .irq_set_affinity() implementation that is no longer needed. Link: https://lore.kernel.org/r/20240723132958.41320-9-marek.vasut+renesas@mailbox.org Signed-off-by: Marek Vasut [bhelgaas: commit log] Signed-off-by: Bjorn Helgaas Reviewed-by: Manivannan Sadhasivam Acked-by: Thomas Gleixner --- drivers/pci/controller/mobiveil/pcie-mobiveil-host.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/drivers/pci/controller/mobiveil/pcie-mobiveil-host.c b/drivers/pci/controller/mobiveil/pcie-mobiveil-host.c index 32951f7d6d6d..0e088e74155d 100644 --- a/drivers/pci/controller/mobiveil/pcie-mobiveil-host.c +++ b/drivers/pci/controller/mobiveil/pcie-mobiveil-host.c @@ -360,8 +360,8 @@ static struct irq_chip mobiveil_msi_irq_chip = { }; static struct msi_domain_info mobiveil_msi_domain_info = { - .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | - MSI_FLAG_PCI_MSIX), + .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_NO_AFFINITY | MSI_FLAG_PCI_MSIX, .chip = &mobiveil_msi_irq_chip, }; @@ -378,16 +378,9 @@ static void mobiveil_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) (int)data->hwirq, msg->address_hi, msg->address_lo); } -static int mobiveil_msi_set_affinity(struct irq_data *irq_data, - const struct cpumask *mask, bool force) -{ - return -EINVAL; -} - static struct irq_chip mobiveil_msi_bottom_irq_chip = { .name = "Mobiveil MSI", .irq_compose_msi_msg = mobiveil_compose_msi_msg, - .irq_set_affinity = mobiveil_msi_set_affinity, }; static int mobiveil_irq_msi_domain_alloc(struct irq_domain *domain, From e934abaac09692201d109b094a01b3dea3915382 Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Tue, 23 Jul 2024 15:27:09 +0200 Subject: [PATCH 013/302] PCI: plda: Silence 'set affinity failed' warning Use MSI_FLAG_NO_AFFINITY, which keeps .irq_set_affinity() unset and allows migrate_one_irq() to exit right away, without warnings like this: IRQ...: set affinity failed(-22) Remove the .irq_set_affinity() implementation that is no longer needed. Link: https://lore.kernel.org/r/20240723132958.41320-10-marek.vasut+renesas@mailbox.org Signed-off-by: Marek Vasut [bhelgaas: commit log] Signed-off-by: Bjorn Helgaas Reviewed-by: Manivannan Sadhasivam Acked-by: Thomas Gleixner --- drivers/pci/controller/plda/pcie-plda-host.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/drivers/pci/controller/plda/pcie-plda-host.c b/drivers/pci/controller/plda/pcie-plda-host.c index a18923d7cea6..8533dc618d45 100644 --- a/drivers/pci/controller/plda/pcie-plda-host.c +++ b/drivers/pci/controller/plda/pcie-plda-host.c @@ -76,17 +76,10 @@ static void plda_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) (int)data->hwirq, msg->address_hi, msg->address_lo); } -static int plda_msi_set_affinity(struct irq_data *irq_data, - const struct cpumask *mask, bool force) -{ - return -EINVAL; -} - static struct irq_chip plda_msi_bottom_irq_chip = { .name = "PLDA MSI", .irq_ack = plda_msi_bottom_irq_ack, .irq_compose_msi_msg = plda_compose_msi_msg, - .irq_set_affinity = plda_msi_set_affinity, }; static int plda_irq_msi_domain_alloc(struct irq_domain *domain, @@ -146,8 +139,8 @@ static struct irq_chip plda_msi_irq_chip = { }; static struct msi_domain_info plda_msi_domain_info = { - .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | - MSI_FLAG_PCI_MSIX), + .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_NO_AFFINITY | MSI_FLAG_PCI_MSIX, .chip = &plda_msi_irq_chip, }; From 57d1992b396d5dbbc70b938c15a10d9d3469c48d Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Tue, 23 Jul 2024 15:27:10 +0200 Subject: [PATCH 014/302] PCI: rcar-host: Silence 'set affinity failed' warning Use MSI_FLAG_NO_AFFINITY, which keeps .irq_set_affinity() unset and allows migrate_one_irq() to exit right away, without warnings like this: IRQ...: set affinity failed(-22) Remove the .irq_set_affinity() implementation that is no longer needed. Link: https://lore.kernel.org/r/20240723132958.41320-11-marek.vasut+renesas@mailbox.org Signed-off-by: Marek Vasut [bhelgaas: commit log] Signed-off-by: Bjorn Helgaas Reviewed-by: Manivannan Sadhasivam Acked-by: Thomas Gleixner --- drivers/pci/controller/pcie-rcar-host.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/drivers/pci/controller/pcie-rcar-host.c b/drivers/pci/controller/pcie-rcar-host.c index c01efc6ea64f..3dd653f3d784 100644 --- a/drivers/pci/controller/pcie-rcar-host.c +++ b/drivers/pci/controller/pcie-rcar-host.c @@ -658,11 +658,6 @@ static void rcar_msi_irq_unmask(struct irq_data *d) spin_unlock_irqrestore(&msi->mask_lock, flags); } -static int rcar_msi_set_affinity(struct irq_data *d, const struct cpumask *mask, bool force) -{ - return -EINVAL; -} - static void rcar_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) { struct rcar_msi *msi = irq_data_get_irq_chip_data(data); @@ -678,7 +673,6 @@ static struct irq_chip rcar_msi_bottom_chip = { .irq_ack = rcar_msi_irq_ack, .irq_mask = rcar_msi_irq_mask, .irq_unmask = rcar_msi_irq_unmask, - .irq_set_affinity = rcar_msi_set_affinity, .irq_compose_msi_msg = rcar_compose_msi_msg, }; @@ -725,8 +719,8 @@ static const struct irq_domain_ops rcar_msi_domain_ops = { }; static struct msi_domain_info rcar_msi_info = { - .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | - MSI_FLAG_MULTI_PCI_MSI), + .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_NO_AFFINITY | MSI_FLAG_MULTI_PCI_MSI, .chip = &rcar_msi_top_chip, }; From 7216311681e7f53845c790945a05a61cbbf471fc Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Tue, 23 Jul 2024 15:27:11 +0200 Subject: [PATCH 015/302] PCI: tegra: Silence 'set affinity failed' warning Use MSI_FLAG_NO_AFFINITY, which keeps .irq_set_affinity() unset and allows migrate_one_irq() to exit right away, without warnings like this: IRQ...: set affinity failed(-22) Remove the .irq_set_affinity() implementation that is no longer needed. Link: https://lore.kernel.org/r/20240723132958.41320-12-marek.vasut+renesas@mailbox.org Signed-off-by: Marek Vasut [bhelgaas: commit log] Signed-off-by: Bjorn Helgaas Reviewed-by: Manivannan Sadhasivam Acked-by: Thomas Gleixner --- drivers/pci/controller/pci-tegra.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/drivers/pci/controller/pci-tegra.c b/drivers/pci/controller/pci-tegra.c index 038d974a318e..d7517c3976e7 100644 --- a/drivers/pci/controller/pci-tegra.c +++ b/drivers/pci/controller/pci-tegra.c @@ -1629,11 +1629,6 @@ static void tegra_msi_irq_unmask(struct irq_data *d) spin_unlock_irqrestore(&msi->mask_lock, flags); } -static int tegra_msi_set_affinity(struct irq_data *d, const struct cpumask *mask, bool force) -{ - return -EINVAL; -} - static void tegra_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) { struct tegra_msi *msi = irq_data_get_irq_chip_data(data); @@ -1648,7 +1643,6 @@ static struct irq_chip tegra_msi_bottom_chip = { .irq_ack = tegra_msi_irq_ack, .irq_mask = tegra_msi_irq_mask, .irq_unmask = tegra_msi_irq_unmask, - .irq_set_affinity = tegra_msi_set_affinity, .irq_compose_msi_msg = tegra_compose_msi_msg, }; @@ -1697,8 +1691,8 @@ static const struct irq_domain_ops tegra_msi_domain_ops = { }; static struct msi_domain_info tegra_msi_info = { - .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | - MSI_FLAG_PCI_MSIX), + .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_NO_AFFINITY | MSI_FLAG_PCI_MSIX, .chip = &tegra_msi_top_chip, }; From 1f35a0c74e441e1a21b5414c25bc01f06e9cca31 Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Fri, 26 Jul 2024 17:29:11 +0800 Subject: [PATCH 016/302] PCI/ACPI: Increase Loongson max PCI hosts to 8 Beginning with Loongson-3C6000, there can be up to 8 PCI hosts for multi-chip machines. To support these machines, increase the number of entries in mcfg_quirks to 8. Link: https://lore.kernel.org/r/20240726092911.2042656-1-chenhuacai@loongson.cn Signed-off-by: Haowei Zheng Signed-off-by: Huacai Chen Signed-off-by: Bjorn Helgaas --- drivers/acpi/pci_mcfg.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/drivers/acpi/pci_mcfg.c b/drivers/acpi/pci_mcfg.c index 860014b89b8e..58e10a980114 100644 --- a/drivers/acpi/pci_mcfg.c +++ b/drivers/acpi/pci_mcfg.c @@ -181,6 +181,18 @@ static struct mcfg_fixup mcfg_quirks[] = { LOONGSON_ECAM_MCFG("LOONGSON", 0), LOONGSON_ECAM_MCFG("\0", 1), LOONGSON_ECAM_MCFG("LOONGSON", 1), + LOONGSON_ECAM_MCFG("\0", 2), + LOONGSON_ECAM_MCFG("LOONGSON", 2), + LOONGSON_ECAM_MCFG("\0", 3), + LOONGSON_ECAM_MCFG("LOONGSON", 3), + LOONGSON_ECAM_MCFG("\0", 4), + LOONGSON_ECAM_MCFG("LOONGSON", 4), + LOONGSON_ECAM_MCFG("\0", 5), + LOONGSON_ECAM_MCFG("LOONGSON", 5), + LOONGSON_ECAM_MCFG("\0", 6), + LOONGSON_ECAM_MCFG("LOONGSON", 6), + LOONGSON_ECAM_MCFG("\0", 7), + LOONGSON_ECAM_MCFG("LOONGSON", 7), #endif /* LOONGARCH */ }; From 5d8491ae3b8083029ed58ae2aba66cdb70617d4d Mon Sep 17 00:00:00 2001 From: ngn Date: Tue, 23 Jul 2024 14:43:25 +0300 Subject: [PATCH 017/302] PCI: shpchp: Remove hpc_ops Remove the hpc_ops struct from shpchp. This struct is unnecessary and no other hotplug controller implements it. A similar thing has already been done in pciehp with 82a9e79ef132 ("PCI: pciehp: remove hpc_ops"). Link: https://lore.kernel.org/r/Zp-XXVW4hlcMASEc@archbtw Signed-off-by: ngn Signed-off-by: Bjorn Helgaas --- drivers/pci/hotplug/TODO | 5 -- drivers/pci/hotplug/shpchp.h | 38 +++++++-------- drivers/pci/hotplug/shpchp_core.c | 15 +++--- drivers/pci/hotplug/shpchp_ctrl.c | 79 +++++++++++++++---------------- drivers/pci/hotplug/shpchp_hpc.c | 63 ++++++++---------------- 5 files changed, 82 insertions(+), 118 deletions(-) diff --git a/drivers/pci/hotplug/TODO b/drivers/pci/hotplug/TODO index 9d428b0ea524..92e6e20e8595 100644 --- a/drivers/pci/hotplug/TODO +++ b/drivers/pci/hotplug/TODO @@ -51,11 +51,6 @@ ibmphp: shpchp: -* There is only a single implementation of struct hpc_ops. Can the struct be - removed and its functions invoked directly? This has already been done in - pciehp with commit 82a9e79ef132 ("PCI: pciehp: remove hpc_ops"). Clarify - if there was a specific reason not to apply the same change to shpchp. - * The hardirq handler shpc_isr() queues events on a workqueue. It can be simplified by converting it to threaded IRQ handling. Use pciehp as a template. diff --git a/drivers/pci/hotplug/shpchp.h b/drivers/pci/hotplug/shpchp.h index 3a97f455336e..f0e2d2d54d71 100644 --- a/drivers/pci/hotplug/shpchp.h +++ b/drivers/pci/hotplug/shpchp.h @@ -72,7 +72,6 @@ struct slot { u8 latch_save; u8 pwr_save; struct controller *ctrl; - const struct hpc_ops *hpc_ops; struct hotplug_slot hotplug_slot; struct list_head slot_list; struct delayed_work work; /* work for button event */ @@ -94,7 +93,6 @@ struct controller { int slot_num_inc; /* 1 or -1 */ struct pci_dev *pci_dev; struct list_head slot_list; - const struct hpc_ops *hpc_ops; wait_queue_head_t queue; /* sleep & wake process */ u8 slot_device_offset; u32 pcix_misc2_reg; /* for amd pogo errata */ @@ -300,24 +298,22 @@ static inline void amd_pogo_errata_restore_misc_reg(struct slot *p_slot) pci_write_config_dword(p_slot->ctrl->pci_dev, PCIX_MISCII_OFFSET, pcix_misc2_temp); } -struct hpc_ops { - int (*power_on_slot)(struct slot *slot); - int (*slot_enable)(struct slot *slot); - int (*slot_disable)(struct slot *slot); - int (*set_bus_speed_mode)(struct slot *slot, enum pci_bus_speed speed); - int (*get_power_status)(struct slot *slot, u8 *status); - int (*get_attention_status)(struct slot *slot, u8 *status); - int (*set_attention_status)(struct slot *slot, u8 status); - int (*get_latch_status)(struct slot *slot, u8 *status); - int (*get_adapter_status)(struct slot *slot, u8 *status); - int (*get_adapter_speed)(struct slot *slot, enum pci_bus_speed *speed); - int (*get_prog_int)(struct slot *slot, u8 *prog_int); - int (*query_power_fault)(struct slot *slot); - void (*green_led_on)(struct slot *slot); - void (*green_led_off)(struct slot *slot); - void (*green_led_blink)(struct slot *slot); - void (*release_ctlr)(struct controller *ctrl); - int (*check_cmd_status)(struct controller *ctrl); -}; +int shpchp_power_on_slot(struct slot *slot); +int shpchp_slot_enable(struct slot *slot); +int shpchp_slot_disable(struct slot *slot); +int shpchp_set_bus_speed_mode(struct slot *slot, enum pci_bus_speed speed); +int shpchp_get_power_status(struct slot *slot, u8 *status); +int shpchp_get_attention_status(struct slot *slot, u8 *status); +int shpchp_set_attention_status(struct slot *slot, u8 status); +int shpchp_get_latch_status(struct slot *slot, u8 *status); +int shpchp_get_adapter_status(struct slot *slot, u8 *status); +int shpchp_get_adapter_speed(struct slot *slot, enum pci_bus_speed *speed); +int shpchp_get_prog_int(struct slot *slot, u8 *prog_int); +int shpchp_query_power_fault(struct slot *slot); +void shpchp_green_led_on(struct slot *slot); +void shpchp_green_led_off(struct slot *slot); +void shpchp_green_led_blink(struct slot *slot); +void shpchp_release_ctlr(struct controller *ctrl); +int shpchp_check_cmd_status(struct controller *ctrl); #endif /* _SHPCHP_H */ diff --git a/drivers/pci/hotplug/shpchp_core.c b/drivers/pci/hotplug/shpchp_core.c index 56c7795ed890..a92e28b72908 100644 --- a/drivers/pci/hotplug/shpchp_core.c +++ b/drivers/pci/hotplug/shpchp_core.c @@ -81,7 +81,6 @@ static int init_slots(struct controller *ctrl) slot->ctrl = ctrl; slot->bus = ctrl->pci_dev->subordinate->number; slot->device = ctrl->slot_device_offset + i; - slot->hpc_ops = ctrl->hpc_ops; slot->number = ctrl->first_slot + (ctrl->slot_num_inc * i); slot->wq = alloc_workqueue("shpchp-%d", 0, 0, slot->number); @@ -150,7 +149,7 @@ static int set_attention_status(struct hotplug_slot *hotplug_slot, u8 status) __func__, slot_name(slot)); slot->attention_save = status; - slot->hpc_ops->set_attention_status(slot, status); + shpchp_set_attention_status(slot, status); return 0; } @@ -183,7 +182,7 @@ static int get_power_status(struct hotplug_slot *hotplug_slot, u8 *value) ctrl_dbg(slot->ctrl, "%s: physical_slot = %s\n", __func__, slot_name(slot)); - retval = slot->hpc_ops->get_power_status(slot, value); + retval = shpchp_get_power_status(slot, value); if (retval < 0) *value = slot->pwr_save; @@ -198,7 +197,7 @@ static int get_attention_status(struct hotplug_slot *hotplug_slot, u8 *value) ctrl_dbg(slot->ctrl, "%s: physical_slot = %s\n", __func__, slot_name(slot)); - retval = slot->hpc_ops->get_attention_status(slot, value); + retval = shpchp_get_attention_status(slot, value); if (retval < 0) *value = slot->attention_save; @@ -213,7 +212,7 @@ static int get_latch_status(struct hotplug_slot *hotplug_slot, u8 *value) ctrl_dbg(slot->ctrl, "%s: physical_slot = %s\n", __func__, slot_name(slot)); - retval = slot->hpc_ops->get_latch_status(slot, value); + retval = shpchp_get_latch_status(slot, value); if (retval < 0) *value = slot->latch_save; @@ -228,7 +227,7 @@ static int get_adapter_status(struct hotplug_slot *hotplug_slot, u8 *value) ctrl_dbg(slot->ctrl, "%s: physical_slot = %s\n", __func__, slot_name(slot)); - retval = slot->hpc_ops->get_adapter_status(slot, value); + retval = shpchp_get_adapter_status(slot, value); if (retval < 0) *value = slot->presence_save; @@ -293,7 +292,7 @@ static int shpc_probe(struct pci_dev *pdev, const struct pci_device_id *ent) err_cleanup_slots: cleanup_slots(ctrl); err_out_release_ctlr: - ctrl->hpc_ops->release_ctlr(ctrl); + shpchp_release_ctlr(ctrl); err_out_free_ctrl: kfree(ctrl); err_out_none: @@ -306,7 +305,7 @@ static void shpc_remove(struct pci_dev *dev) dev->shpc_managed = 0; shpchp_remove_ctrl_files(ctrl); - ctrl->hpc_ops->release_ctlr(ctrl); + shpchp_release_ctlr(ctrl); kfree(ctrl); } diff --git a/drivers/pci/hotplug/shpchp_ctrl.c b/drivers/pci/hotplug/shpchp_ctrl.c index 6a6705e0cf17..e6c6f23bae27 100644 --- a/drivers/pci/hotplug/shpchp_ctrl.c +++ b/drivers/pci/hotplug/shpchp_ctrl.c @@ -51,7 +51,7 @@ u8 shpchp_handle_attention_button(u8 hp_slot, struct controller *ctrl) ctrl_dbg(ctrl, "Attention button interrupt received\n"); p_slot = shpchp_find_slot(ctrl, hp_slot + ctrl->slot_device_offset); - p_slot->hpc_ops->get_adapter_status(p_slot, &(p_slot->presence_save)); + shpchp_get_adapter_status(p_slot, &p_slot->presence_save); /* * Button pressed - See if need to TAKE ACTION!!! @@ -75,8 +75,8 @@ u8 shpchp_handle_switch_change(u8 hp_slot, struct controller *ctrl) ctrl_dbg(ctrl, "Switch interrupt received\n"); p_slot = shpchp_find_slot(ctrl, hp_slot + ctrl->slot_device_offset); - p_slot->hpc_ops->get_adapter_status(p_slot, &(p_slot->presence_save)); - p_slot->hpc_ops->get_latch_status(p_slot, &getstatus); + shpchp_get_adapter_status(p_slot, &p_slot->presence_save); + shpchp_get_latch_status(p_slot, &getstatus); ctrl_dbg(ctrl, "Card present %x Power status %x\n", p_slot->presence_save, p_slot->pwr_save); @@ -116,7 +116,7 @@ u8 shpchp_handle_presence_change(u8 hp_slot, struct controller *ctrl) /* * Save the presence state */ - p_slot->hpc_ops->get_adapter_status(p_slot, &(p_slot->presence_save)); + shpchp_get_adapter_status(p_slot, &p_slot->presence_save); if (p_slot->presence_save) { /* * Card Present @@ -148,7 +148,7 @@ u8 shpchp_handle_power_fault(u8 hp_slot, struct controller *ctrl) p_slot = shpchp_find_slot(ctrl, hp_slot + ctrl->slot_device_offset); - if (!(p_slot->hpc_ops->query_power_fault(p_slot))) { + if (!(shpchp_query_power_fault(p_slot))) { /* * Power fault Cleared */ @@ -181,7 +181,7 @@ static int change_bus_speed(struct controller *ctrl, struct slot *p_slot, int rc = 0; ctrl_dbg(ctrl, "Change speed to %d\n", speed); - rc = p_slot->hpc_ops->set_bus_speed_mode(p_slot, speed); + rc = shpchp_set_bus_speed_mode(p_slot, speed); if (rc) { ctrl_err(ctrl, "%s: Issue of set bus speed mode command failed\n", __func__); @@ -241,14 +241,14 @@ static int board_added(struct slot *p_slot) __func__, p_slot->device, ctrl->slot_device_offset, hp_slot); /* Power on slot without connecting to bus */ - rc = p_slot->hpc_ops->power_on_slot(p_slot); + rc = shpchp_power_on_slot(p_slot); if (rc) { ctrl_err(ctrl, "Failed to power on slot\n"); return -1; } if ((ctrl->pci_dev->vendor == 0x8086) && (ctrl->pci_dev->device == 0x0332)) { - rc = p_slot->hpc_ops->set_bus_speed_mode(p_slot, PCI_SPEED_33MHz); + rc = shpchp_set_bus_speed_mode(p_slot, PCI_SPEED_33MHz); if (rc) { ctrl_err(ctrl, "%s: Issue of set bus speed mode command failed\n", __func__); @@ -256,14 +256,14 @@ static int board_added(struct slot *p_slot) } /* turn on board, blink green LED, turn off Amber LED */ - rc = p_slot->hpc_ops->slot_enable(p_slot); + rc = shpchp_slot_enable(p_slot); if (rc) { ctrl_err(ctrl, "Issue of Slot Enable command failed\n"); return rc; } } - rc = p_slot->hpc_ops->get_adapter_speed(p_slot, &asp); + rc = shpchp_get_adapter_speed(p_slot, &asp); if (rc) { ctrl_err(ctrl, "Can't get adapter speed or bus mode mismatch\n"); return WRONG_BUS_FREQUENCY; @@ -285,7 +285,7 @@ static int board_added(struct slot *p_slot) return rc; /* turn on board, blink green LED, turn off Amber LED */ - rc = p_slot->hpc_ops->slot_enable(p_slot); + rc = shpchp_slot_enable(p_slot); if (rc) { ctrl_err(ctrl, "Issue of Slot Enable command failed\n"); return rc; @@ -313,13 +313,13 @@ static int board_added(struct slot *p_slot) p_slot->is_a_board = 0x01; p_slot->pwr_save = 1; - p_slot->hpc_ops->green_led_on(p_slot); + shpchp_green_led_on(p_slot); return 0; err_exit: /* turn off slot, turn on Amber LED, turn off Green LED */ - rc = p_slot->hpc_ops->slot_disable(p_slot); + rc = shpchp_slot_disable(p_slot); if (rc) { ctrl_err(ctrl, "%s: Issue of Slot Disable command failed\n", __func__); @@ -352,14 +352,14 @@ static int remove_board(struct slot *p_slot) p_slot->status = 0x01; /* turn off slot, turn on Amber LED, turn off Green LED */ - rc = p_slot->hpc_ops->slot_disable(p_slot); + rc = shpchp_slot_disable(p_slot); if (rc) { ctrl_err(ctrl, "%s: Issue of Slot Disable command failed\n", __func__); return rc; } - rc = p_slot->hpc_ops->set_attention_status(p_slot, 0); + rc = shpchp_set_attention_status(p_slot, 0); if (rc) { ctrl_err(ctrl, "Issue of Set Attention command failed\n"); return rc; @@ -401,7 +401,7 @@ static void shpchp_pushbutton_thread(struct work_struct *work) case POWERON_STATE: mutex_unlock(&p_slot->lock); if (shpchp_enable_slot(p_slot)) - p_slot->hpc_ops->green_led_off(p_slot); + shpchp_green_led_off(p_slot); mutex_lock(&p_slot->lock); p_slot->state = STATIC_STATE; break; @@ -446,10 +446,10 @@ void shpchp_queue_pushbutton_work(struct work_struct *work) static void update_slot_info(struct slot *slot) { - slot->hpc_ops->get_power_status(slot, &slot->pwr_save); - slot->hpc_ops->get_attention_status(slot, &slot->attention_save); - slot->hpc_ops->get_latch_status(slot, &slot->latch_save); - slot->hpc_ops->get_adapter_status(slot, &slot->presence_save); + shpchp_get_power_status(slot, &slot->pwr_save); + shpchp_get_attention_status(slot, &slot->attention_save); + shpchp_get_latch_status(slot, &slot->latch_save); + shpchp_get_adapter_status(slot, &slot->presence_save); } /* @@ -462,7 +462,7 @@ static void handle_button_press_event(struct slot *p_slot) switch (p_slot->state) { case STATIC_STATE: - p_slot->hpc_ops->get_power_status(p_slot, &getstatus); + shpchp_get_power_status(p_slot, &getstatus); if (getstatus) { p_slot->state = BLINKINGOFF_STATE; ctrl_info(ctrl, "PCI slot #%s - powering off due to button press\n", @@ -473,8 +473,8 @@ static void handle_button_press_event(struct slot *p_slot) slot_name(p_slot)); } /* blink green LED and turn off amber */ - p_slot->hpc_ops->green_led_blink(p_slot); - p_slot->hpc_ops->set_attention_status(p_slot, 0); + shpchp_green_led_blink(p_slot); + shpchp_set_attention_status(p_slot, 0); queue_delayed_work(p_slot->wq, &p_slot->work, 5*HZ); break; @@ -489,10 +489,10 @@ static void handle_button_press_event(struct slot *p_slot) slot_name(p_slot)); cancel_delayed_work(&p_slot->work); if (p_slot->state == BLINKINGOFF_STATE) - p_slot->hpc_ops->green_led_on(p_slot); + shpchp_green_led_on(p_slot); else - p_slot->hpc_ops->green_led_off(p_slot); - p_slot->hpc_ops->set_attention_status(p_slot, 0); + shpchp_green_led_off(p_slot); + shpchp_set_attention_status(p_slot, 0); ctrl_info(ctrl, "PCI slot #%s - action canceled due to button press\n", slot_name(p_slot)); p_slot->state = STATIC_STATE; @@ -526,8 +526,8 @@ static void interrupt_event_handler(struct work_struct *work) break; case INT_POWER_FAULT: ctrl_dbg(p_slot->ctrl, "%s: Power fault\n", __func__); - p_slot->hpc_ops->set_attention_status(p_slot, 1); - p_slot->hpc_ops->green_led_off(p_slot); + shpchp_set_attention_status(p_slot, 1); + shpchp_green_led_off(p_slot); break; default: update_slot_info(p_slot); @@ -547,17 +547,17 @@ static int shpchp_enable_slot (struct slot *p_slot) /* Check to see if (latch closed, card present, power off) */ mutex_lock(&p_slot->ctrl->crit_sect); - rc = p_slot->hpc_ops->get_adapter_status(p_slot, &getstatus); + rc = shpchp_get_adapter_status(p_slot, &getstatus); if (rc || !getstatus) { ctrl_info(ctrl, "No adapter on slot(%s)\n", slot_name(p_slot)); goto out; } - rc = p_slot->hpc_ops->get_latch_status(p_slot, &getstatus); + rc = shpchp_get_latch_status(p_slot, &getstatus); if (rc || getstatus) { ctrl_info(ctrl, "Latch open on slot(%s)\n", slot_name(p_slot)); goto out; } - rc = p_slot->hpc_ops->get_power_status(p_slot, &getstatus); + rc = shpchp_get_power_status(p_slot, &getstatus); if (rc || getstatus) { ctrl_info(ctrl, "Already enabled on slot(%s)\n", slot_name(p_slot)); @@ -567,10 +567,10 @@ static int shpchp_enable_slot (struct slot *p_slot) p_slot->is_a_board = 1; /* We have to save the presence info for these slots */ - p_slot->hpc_ops->get_adapter_status(p_slot, &(p_slot->presence_save)); - p_slot->hpc_ops->get_power_status(p_slot, &(p_slot->pwr_save)); + shpchp_get_adapter_status(p_slot, &p_slot->presence_save); + shpchp_get_power_status(p_slot, &p_slot->pwr_save); ctrl_dbg(ctrl, "%s: p_slot->pwr_save %x\n", __func__, p_slot->pwr_save); - p_slot->hpc_ops->get_latch_status(p_slot, &getstatus); + shpchp_get_latch_status(p_slot, &getstatus); if ((p_slot->ctrl->pci_dev->vendor == PCI_VENDOR_ID_AMD && p_slot->ctrl->pci_dev->device == PCI_DEVICE_ID_AMD_POGO_7458) @@ -584,9 +584,8 @@ static int shpchp_enable_slot (struct slot *p_slot) retval = board_added(p_slot); if (retval) { - p_slot->hpc_ops->get_adapter_status(p_slot, - &(p_slot->presence_save)); - p_slot->hpc_ops->get_latch_status(p_slot, &getstatus); + shpchp_get_adapter_status(p_slot, &p_slot->presence_save); + shpchp_get_latch_status(p_slot, &getstatus); } update_slot_info(p_slot); @@ -608,17 +607,17 @@ static int shpchp_disable_slot (struct slot *p_slot) /* Check to see if (latch closed, card present, power on) */ mutex_lock(&p_slot->ctrl->crit_sect); - rc = p_slot->hpc_ops->get_adapter_status(p_slot, &getstatus); + rc = shpchp_get_adapter_status(p_slot, &getstatus); if (rc || !getstatus) { ctrl_info(ctrl, "No adapter on slot(%s)\n", slot_name(p_slot)); goto out; } - rc = p_slot->hpc_ops->get_latch_status(p_slot, &getstatus); + rc = shpchp_get_latch_status(p_slot, &getstatus); if (rc || getstatus) { ctrl_info(ctrl, "Latch open on slot(%s)\n", slot_name(p_slot)); goto out; } - rc = p_slot->hpc_ops->get_power_status(p_slot, &getstatus); + rc = shpchp_get_power_status(p_slot, &getstatus); if (rc || !getstatus) { ctrl_info(ctrl, "Already disabled on slot(%s)\n", slot_name(p_slot)); diff --git a/drivers/pci/hotplug/shpchp_hpc.c b/drivers/pci/hotplug/shpchp_hpc.c index 48e4daefc44a..012b9e3fe5b0 100644 --- a/drivers/pci/hotplug/shpchp_hpc.c +++ b/drivers/pci/hotplug/shpchp_hpc.c @@ -167,7 +167,6 @@ static irqreturn_t shpc_isr(int irq, void *dev_id); static void start_int_poll_timer(struct controller *ctrl, int sec); -static int hpc_check_cmd_status(struct controller *ctrl); static inline u8 shpc_readb(struct controller *ctrl, int reg) { @@ -317,7 +316,7 @@ static int shpc_write_cmd(struct slot *slot, u8 t_slot, u8 cmd) if (retval) goto out; - cmd_status = hpc_check_cmd_status(slot->ctrl); + cmd_status = shpchp_check_cmd_status(slot->ctrl); if (cmd_status) { ctrl_err(ctrl, "Failed to issued command 0x%x (error code = %d)\n", cmd, cmd_status); @@ -328,7 +327,7 @@ static int shpc_write_cmd(struct slot *slot, u8 t_slot, u8 cmd) return retval; } -static int hpc_check_cmd_status(struct controller *ctrl) +int shpchp_check_cmd_status(struct controller *ctrl) { int retval = 0; u16 cmd_status = shpc_readw(ctrl, CMD_STATUS) & 0x000F; @@ -357,7 +356,7 @@ static int hpc_check_cmd_status(struct controller *ctrl) } -static int hpc_get_attention_status(struct slot *slot, u8 *status) +int shpchp_get_attention_status(struct slot *slot, u8 *status) { struct controller *ctrl = slot->ctrl; u32 slot_reg = shpc_readl(ctrl, SLOT_REG(slot->hp_slot)); @@ -381,7 +380,7 @@ static int hpc_get_attention_status(struct slot *slot, u8 *status) return 0; } -static int hpc_get_power_status(struct slot *slot, u8 *status) +int shpchp_get_power_status(struct slot *slot, u8 *status) { struct controller *ctrl = slot->ctrl; u32 slot_reg = shpc_readl(ctrl, SLOT_REG(slot->hp_slot)); @@ -406,7 +405,7 @@ static int hpc_get_power_status(struct slot *slot, u8 *status) } -static int hpc_get_latch_status(struct slot *slot, u8 *status) +int shpchp_get_latch_status(struct slot *slot, u8 *status) { struct controller *ctrl = slot->ctrl; u32 slot_reg = shpc_readl(ctrl, SLOT_REG(slot->hp_slot)); @@ -416,7 +415,7 @@ static int hpc_get_latch_status(struct slot *slot, u8 *status) return 0; } -static int hpc_get_adapter_status(struct slot *slot, u8 *status) +int shpchp_get_adapter_status(struct slot *slot, u8 *status) { struct controller *ctrl = slot->ctrl; u32 slot_reg = shpc_readl(ctrl, SLOT_REG(slot->hp_slot)); @@ -427,7 +426,7 @@ static int hpc_get_adapter_status(struct slot *slot, u8 *status) return 0; } -static int hpc_get_prog_int(struct slot *slot, u8 *prog_int) +int shpchp_get_prog_int(struct slot *slot, u8 *prog_int) { struct controller *ctrl = slot->ctrl; @@ -436,7 +435,7 @@ static int hpc_get_prog_int(struct slot *slot, u8 *prog_int) return 0; } -static int hpc_get_adapter_speed(struct slot *slot, enum pci_bus_speed *value) +int shpchp_get_adapter_speed(struct slot *slot, enum pci_bus_speed *value) { int retval = 0; struct controller *ctrl = slot->ctrl; @@ -444,7 +443,7 @@ static int hpc_get_adapter_speed(struct slot *slot, enum pci_bus_speed *value) u8 m66_cap = !!(slot_reg & MHZ66_CAP); u8 pi, pcix_cap; - retval = hpc_get_prog_int(slot, &pi); + retval = shpchp_get_prog_int(slot, &pi); if (retval) return retval; @@ -489,7 +488,7 @@ static int hpc_get_adapter_speed(struct slot *slot, enum pci_bus_speed *value) return retval; } -static int hpc_query_power_fault(struct slot *slot) +int shpchp_query_power_fault(struct slot *slot) { struct controller *ctrl = slot->ctrl; u32 slot_reg = shpc_readl(ctrl, SLOT_REG(slot->hp_slot)); @@ -498,7 +497,7 @@ static int hpc_query_power_fault(struct slot *slot) return !(slot_reg & POWER_FAULT); } -static int hpc_set_attention_status(struct slot *slot, u8 value) +int shpchp_set_attention_status(struct slot *slot, u8 value) { u8 slot_cmd = 0; @@ -520,22 +519,22 @@ static int hpc_set_attention_status(struct slot *slot, u8 value) } -static void hpc_set_green_led_on(struct slot *slot) +void shpchp_green_led_on(struct slot *slot) { shpc_write_cmd(slot, slot->hp_slot, SET_PWR_ON); } -static void hpc_set_green_led_off(struct slot *slot) +void shpchp_green_led_off(struct slot *slot) { shpc_write_cmd(slot, slot->hp_slot, SET_PWR_OFF); } -static void hpc_set_green_led_blink(struct slot *slot) +void shpchp_green_led_blink(struct slot *slot) { shpc_write_cmd(slot, slot->hp_slot, SET_PWR_BLINK); } -static void hpc_release_ctlr(struct controller *ctrl) +void shpchp_release_ctlr(struct controller *ctrl) { int i; u32 slot_reg, serr_int; @@ -575,7 +574,7 @@ static void hpc_release_ctlr(struct controller *ctrl) release_mem_region(ctrl->mmio_base, ctrl->mmio_size); } -static int hpc_power_on_slot(struct slot *slot) +int shpchp_power_on_slot(struct slot *slot) { int retval; @@ -586,7 +585,7 @@ static int hpc_power_on_slot(struct slot *slot) return retval; } -static int hpc_slot_enable(struct slot *slot) +int shpchp_slot_enable(struct slot *slot) { int retval; @@ -599,7 +598,7 @@ static int hpc_slot_enable(struct slot *slot) return retval; } -static int hpc_slot_disable(struct slot *slot) +int shpchp_slot_disable(struct slot *slot) { int retval; @@ -681,7 +680,7 @@ static int shpc_get_cur_bus_speed(struct controller *ctrl) } -static int hpc_set_bus_speed_mode(struct slot *slot, enum pci_bus_speed value) +int shpchp_set_bus_speed_mode(struct slot *slot, enum pci_bus_speed value) { int retval; struct controller *ctrl = slot->ctrl; @@ -871,28 +870,6 @@ static int shpc_get_max_bus_speed(struct controller *ctrl) return retval; } -static const struct hpc_ops shpchp_hpc_ops = { - .power_on_slot = hpc_power_on_slot, - .slot_enable = hpc_slot_enable, - .slot_disable = hpc_slot_disable, - .set_bus_speed_mode = hpc_set_bus_speed_mode, - .set_attention_status = hpc_set_attention_status, - .get_power_status = hpc_get_power_status, - .get_attention_status = hpc_get_attention_status, - .get_latch_status = hpc_get_latch_status, - .get_adapter_status = hpc_get_adapter_status, - - .get_adapter_speed = hpc_get_adapter_speed, - .get_prog_int = hpc_get_prog_int, - - .query_power_fault = hpc_query_power_fault, - .green_led_on = hpc_set_green_led_on, - .green_led_off = hpc_set_green_led_off, - .green_led_blink = hpc_set_green_led_blink, - - .release_ctlr = hpc_release_ctlr, -}; - int shpc_init(struct controller *ctrl, struct pci_dev *pdev) { int rc = -1, num_slots = 0; @@ -978,8 +955,6 @@ int shpc_init(struct controller *ctrl, struct pci_dev *pdev) /* Setup wait queue */ init_waitqueue_head(&ctrl->queue); - ctrl->hpc_ops = &shpchp_hpc_ops; - /* Return PCI Controller Info */ slot_config = shpc_readl(ctrl, SLOT_CONFIG); ctrl->slot_device_offset = (slot_config & FIRST_DEV_NUM) >> 8; From 647e9651a0110fb0ff163ef6bf1318b30f90a08c Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Tue, 23 Jul 2024 15:27:12 +0200 Subject: [PATCH 018/302] PCI: vmd: Silence 'set affinity failed' warning Use MSI_FLAG_NO_AFFINITY, which keeps .irq_set_affinity() unset and allows migrate_one_irq() to exit right away, without warnings like this: IRQ...: set affinity failed(-22) Remove the .irq_set_affinity() implementation that is no longer needed. Link: https://lore.kernel.org/r/20240723132958.41320-13-marek.vasut+renesas@mailbox.org Signed-off-by: Marek Vasut [bhelgaas: commit log] Signed-off-by: Bjorn Helgaas Reviewed-by: Manivannan Sadhasivam Reviewed-by: Nirmal Patel Acked-by: Thomas Gleixner --- drivers/pci/controller/vmd.c | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/drivers/pci/controller/vmd.c b/drivers/pci/controller/vmd.c index a726de0af011..bc849b0d9e8d 100644 --- a/drivers/pci/controller/vmd.c +++ b/drivers/pci/controller/vmd.c @@ -204,22 +204,11 @@ static void vmd_irq_disable(struct irq_data *data) raw_spin_unlock_irqrestore(&list_lock, flags); } -/* - * XXX: Stubbed until we develop acceptable way to not create conflicts with - * other devices sharing the same vector. - */ -static int vmd_irq_set_affinity(struct irq_data *data, - const struct cpumask *dest, bool force) -{ - return -EINVAL; -} - static struct irq_chip vmd_msi_controller = { .name = "VMD-MSI", .irq_enable = vmd_irq_enable, .irq_disable = vmd_irq_disable, .irq_compose_msi_msg = vmd_compose_msi_msg, - .irq_set_affinity = vmd_irq_set_affinity, }; static irq_hw_number_t vmd_get_hwirq(struct msi_domain_info *info, @@ -326,7 +315,7 @@ static struct msi_domain_ops vmd_msi_domain_ops = { static struct msi_domain_info vmd_msi_domain_info = { .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | - MSI_FLAG_PCI_MSIX, + MSI_FLAG_NO_AFFINITY | MSI_FLAG_PCI_MSIX, .ops = &vmd_msi_domain_ops, .chip = &vmd_msi_controller, }; From d5bba5b9ed233ed9d42e0b43e2b4cddcd91b5d7b Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Tue, 23 Jul 2024 15:27:13 +0200 Subject: [PATCH 019/302] PCI: xilinx-nwl: Silence 'set affinity failed' warning Use MSI_FLAG_NO_AFFINITY, which keeps .irq_set_affinity() unset and allows migrate_one_irq() to exit right away, without warnings like this: IRQ...: set affinity failed(-22) Remove the .irq_set_affinity() implementation that is no longer needed. Link: https://lore.kernel.org/r/20240723132958.41320-14-marek.vasut+renesas@mailbox.org Signed-off-by: Marek Vasut [bhelgaas: commit log] Signed-off-by: Bjorn Helgaas Reviewed-by: Manivannan Sadhasivam Acked-by: Thomas Gleixner --- drivers/pci/controller/pcie-xilinx-nwl.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/drivers/pci/controller/pcie-xilinx-nwl.c b/drivers/pci/controller/pcie-xilinx-nwl.c index 0408f4d612b5..d5615cf8563f 100644 --- a/drivers/pci/controller/pcie-xilinx-nwl.c +++ b/drivers/pci/controller/pcie-xilinx-nwl.c @@ -425,8 +425,8 @@ static struct irq_chip nwl_msi_irq_chip = { }; static struct msi_domain_info nwl_msi_domain_info = { - .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | - MSI_FLAG_MULTI_PCI_MSI), + .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_NO_AFFINITY | MSI_FLAG_MULTI_PCI_MSI, .chip = &nwl_msi_irq_chip, }; #endif @@ -441,16 +441,9 @@ static void nwl_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) msg->data = data->hwirq; } -static int nwl_msi_set_affinity(struct irq_data *irq_data, - const struct cpumask *mask, bool force) -{ - return -EINVAL; -} - static struct irq_chip nwl_irq_chip = { .name = "Xilinx MSI", .irq_compose_msi_msg = nwl_compose_msi_msg, - .irq_set_affinity = nwl_msi_set_affinity, }; static int nwl_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, From 9200f2099f801c580c447d89ebb6b671474c0d52 Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Tue, 23 Jul 2024 15:27:14 +0200 Subject: [PATCH 020/302] PCI: xilinx-xdma: Silence 'set affinity failed' warning Use MSI_FLAG_NO_AFFINITY, which keeps .irq_set_affinity() unset and allows migrate_one_irq() to exit right away, without warnings like this: IRQ...: set affinity failed(-22) Remove the .irq_set_affinity() implementation that is no longer needed. Link: https://lore.kernel.org/r/20240723132958.41320-15-marek.vasut+renesas@mailbox.org Signed-off-by: Marek Vasut [bhelgaas: commit log] Signed-off-by: Bjorn Helgaas Reviewed-by: Manivannan Sadhasivam Acked-by: Thomas Gleixner --- drivers/pci/controller/pcie-xilinx-dma-pl.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/drivers/pci/controller/pcie-xilinx-dma-pl.c b/drivers/pci/controller/pcie-xilinx-dma-pl.c index 5be5dfd8398f..ca9044d694da 100644 --- a/drivers/pci/controller/pcie-xilinx-dma-pl.c +++ b/drivers/pci/controller/pcie-xilinx-dma-pl.c @@ -355,8 +355,8 @@ static struct irq_chip xilinx_msi_irq_chip = { }; static struct msi_domain_info xilinx_msi_domain_info = { - .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | - MSI_FLAG_MULTI_PCI_MSI), + .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_NO_AFFINITY | MSI_FLAG_MULTI_PCI_MSI, .chip = &xilinx_msi_irq_chip, }; @@ -370,16 +370,9 @@ static void xilinx_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) msg->data = data->hwirq; } -static int xilinx_msi_set_affinity(struct irq_data *irq_data, - const struct cpumask *mask, bool force) -{ - return -EINVAL; -} - static struct irq_chip xilinx_irq_chip = { .name = "pl_dma:MSI", .irq_compose_msi_msg = xilinx_compose_msi_msg, - .irq_set_affinity = xilinx_msi_set_affinity, }; static int xilinx_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, From abd9b9d94bc604e09ca73ad232c473006f1793d9 Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Tue, 23 Jul 2024 15:27:15 +0200 Subject: [PATCH 021/302] PCI: xilinx: Silence 'set affinity failed' warning Use MSI_FLAG_NO_AFFINITY, which keeps .irq_set_affinity() unset and allows migrate_one_irq() to exit right away, without warnings like this: IRQ...: set affinity failed(-22) Remove the .irq_set_affinity() implementation that is no longer needed. Link: https://lore.kernel.org/r/20240723132958.41320-16-marek.vasut+renesas@mailbox.org Signed-off-by: Marek Vasut [bhelgaas: commit log] Signed-off-by: Bjorn Helgaas Reviewed-by: Manivannan Sadhasivam Acked-by: Thomas Gleixner --- drivers/pci/controller/pcie-xilinx.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/drivers/pci/controller/pcie-xilinx.c b/drivers/pci/controller/pcie-xilinx.c index cb6e9f7b0152..0b534f73a942 100644 --- a/drivers/pci/controller/pcie-xilinx.c +++ b/drivers/pci/controller/pcie-xilinx.c @@ -208,11 +208,6 @@ static struct irq_chip xilinx_msi_top_chip = { .irq_ack = xilinx_msi_top_irq_ack, }; -static int xilinx_msi_set_affinity(struct irq_data *d, const struct cpumask *mask, bool force) -{ - return -EINVAL; -} - static void xilinx_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) { struct xilinx_pcie *pcie = irq_data_get_irq_chip_data(data); @@ -225,7 +220,6 @@ static void xilinx_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) static struct irq_chip xilinx_msi_bottom_chip = { .name = "Xilinx MSI", - .irq_set_affinity = xilinx_msi_set_affinity, .irq_compose_msi_msg = xilinx_compose_msi_msg, }; @@ -271,7 +265,8 @@ static const struct irq_domain_ops xilinx_msi_domain_ops = { }; static struct msi_domain_info xilinx_msi_info = { - .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS), + .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_NO_AFFINITY, .chip = &xilinx_msi_top_chip, }; From 87d5403378cccc557af9e02a8a2c8587ad8b7e9a Mon Sep 17 00:00:00 2001 From: weiyufeng Date: Tue, 6 Aug 2024 14:50:50 +0800 Subject: [PATCH 022/302] PCI: cpqphp: Use PCI_POSSIBLE_ERROR() to check config reads Use PCI_POSSIBLE_ERROR() to check the response we get when we read data from hardware. This unifies PCI error response checking and makes error checks consistent and easier to find. Link: https://lore.kernel.org/r/20240806065050.28725-1-412574090@163.com Signed-off-by: weiyufeng Signed-off-by: Bjorn Helgaas --- drivers/pci/hotplug/cpqphp_pci.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/pci/hotplug/cpqphp_pci.c b/drivers/pci/hotplug/cpqphp_pci.c index e9f1fb333a71..718bc6cf12cb 100644 --- a/drivers/pci/hotplug/cpqphp_pci.c +++ b/drivers/pci/hotplug/cpqphp_pci.c @@ -138,7 +138,7 @@ static int PCI_RefinedAccessConfig(struct pci_bus *bus, unsigned int devfn, u8 o if (pci_bus_read_config_dword(bus, devfn, PCI_VENDOR_ID, &vendID) == -1) return -1; - if (vendID == 0xffffffff) + if (PCI_POSSIBLE_ERROR(vendID)) return -1; return pci_bus_read_config_dword(bus, devfn, offset, value); } @@ -253,7 +253,7 @@ static int PCI_GetBusDevHelper(struct controller *ctrl, u8 *bus_num, u8 *dev_num *dev_num = tdevice; ctrl->pci_bus->number = tbus; pci_bus_read_config_dword(ctrl->pci_bus, *dev_num, PCI_VENDOR_ID, &work); - if (!nobridge || (work == 0xffffffff)) + if (!nobridge || PCI_POSSIBLE_ERROR(work)) return 0; dbg("bus_num %d devfn %d\n", *bus_num, *dev_num); From f73286f3922f9968e850fe1e0e476d04dde2e57c Mon Sep 17 00:00:00 2001 From: Frank Li Date: Thu, 4 Jul 2024 12:40:19 -0400 Subject: [PATCH 023/302] dt-bindings: PCI: host-generic-pci: Drop minItems and maxItems of ranges The ranges description states that "at least one non-prefetchable memory and one or both of prefetchable memory and IO space may also be provided." However, it should not limit the maximum number of ranges to 3. Freescale LS1028 and iMX95 use more than 3 ranges because the space splits some discontinuous prefetchable and non-prefetchable segments. Drop minItems and maxItems. The number of entries will be limited to 32 in pci-bus-common.yaml in dtschema, which should be sufficient. Fixes this CHECK_DTBS warning: arch/arm64/boot/dts/freescale/fsl-ls1028a-rdb.dtb: pcie@1f0000000: ranges: [[2181038080, 1, 4160749568, 1, 4160749568, 0, 1441792], [3254779904, 1, 4162191360, 1, 4162191360, 0, 458752], [2181038080, 1, 4162650112, 1, 4162650112, 0, 131072], [3254779904, 1, 4162781184, 1, 4162781184, 0, 131072], [2181038080, 1, 4162912256, 1, 4162912256, 0, 131072], [3254779904, 1, 4163043328, 1, 4163043328, 0, 131072], [2181038080, 1, 4227858432, 1, 4227858432, 0, 4194304]] is too long Link: https://lore.kernel.org/r/20240704164019.611454-1-Frank.Li@nxp.com Signed-off-by: Frank Li Signed-off-by: Bjorn Helgaas Reviewed-by: Rob Herring (Arm) --- Documentation/devicetree/bindings/pci/host-generic-pci.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/Documentation/devicetree/bindings/pci/host-generic-pci.yaml b/Documentation/devicetree/bindings/pci/host-generic-pci.yaml index bcfbaf5582cc..420d551e9af9 100644 --- a/Documentation/devicetree/bindings/pci/host-generic-pci.yaml +++ b/Documentation/devicetree/bindings/pci/host-generic-pci.yaml @@ -102,8 +102,6 @@ properties: As described in IEEE Std 1275-1994, but must provide at least a definition of non-prefetchable memory. One or both of prefetchable Memory and IO Space may also be provided. - minItems: 1 - maxItems: 3 dma-coherent: true iommu-map: true From 265baca69a0735b64227a43d4be865a95ba514bb Mon Sep 17 00:00:00 2001 From: Lukas Wunner Date: Mon, 5 Aug 2024 17:24:05 +0200 Subject: [PATCH 024/302] s390/pci: Stop usurping pdev->dev.groups Bjorn suggests using pdev->dev.groups for attribute_groups constructed on PCI device enumeration: "Is it feasible to build an attribute group in pci_doe_init() and add it to dev->groups so device_add() will automatically add them?" https://lore.kernel.org/r/20231019165829.GA1381099@bhelgaas Unfortunately on s390, pcibios_device_add() usurps pdev->dev.groups for arch-specific attribute_groups, preventing its use for anything else. Introduce an ARCH_PCI_DEV_GROUPS macro which arches can define in . The macro is visible in drivers/pci/pci-sysfs.c through the inclusion of , which in turn includes . On s390, define the macro to the three attribute_groups previously assigned to pdev->dev.groups. Thereby pdev->dev.groups is made available for use by the PCI core. As a side effect, arch/s390/pci/pci_sysfs.c no longer needs to be compiled into the kernel if CONFIG_SYSFS=n. Link: https://lore.kernel.org/r/7b970f7923e373d1b23784721208f93418720485.1722870934.git.lukas@wunner.de Signed-off-by: Lukas Wunner Signed-off-by: Bjorn Helgaas Acked-by: Niklas Schnelle --- arch/s390/include/asm/pci.h | 9 ++++++++- arch/s390/pci/Makefile | 3 ++- arch/s390/pci/pci.c | 1 - arch/s390/pci/pci_sysfs.c | 14 ++++---------- drivers/pci/pci-sysfs.c | 5 +++++ 5 files changed, 19 insertions(+), 13 deletions(-) diff --git a/arch/s390/include/asm/pci.h b/arch/s390/include/asm/pci.h index 30820a649e6e..9d920ced6047 100644 --- a/arch/s390/include/asm/pci.h +++ b/arch/s390/include/asm/pci.h @@ -191,7 +191,14 @@ static inline bool zdev_enabled(struct zpci_dev *zdev) return (zdev->fh & (1UL << 31)) ? true : false; } -extern const struct attribute_group *zpci_attr_groups[]; +extern const struct attribute_group zpci_attr_group; +extern const struct attribute_group pfip_attr_group; +extern const struct attribute_group zpci_ident_attr_group; + +#define ARCH_PCI_DEV_GROUPS &zpci_attr_group, \ + &pfip_attr_group, \ + &zpci_ident_attr_group, + extern unsigned int s390_pci_force_floating __initdata; extern unsigned int s390_pci_no_rid; diff --git a/arch/s390/pci/Makefile b/arch/s390/pci/Makefile index 0547a10406e7..2c21f0394c9a 100644 --- a/arch/s390/pci/Makefile +++ b/arch/s390/pci/Makefile @@ -3,7 +3,8 @@ # Makefile for the s390 PCI subsystem. # -obj-$(CONFIG_PCI) += pci.o pci_irq.o pci_clp.o pci_sysfs.o \ +obj-$(CONFIG_PCI) += pci.o pci_irq.o pci_clp.o \ pci_event.o pci_debug.o pci_insn.o pci_mmio.o \ pci_bus.o pci_kvm_hook.o obj-$(CONFIG_PCI_IOV) += pci_iov.o +obj-$(CONFIG_SYSFS) += pci_sysfs.o diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c index cff4838fad21..bd9624c20b80 100644 --- a/arch/s390/pci/pci.c +++ b/arch/s390/pci/pci.c @@ -587,7 +587,6 @@ int pcibios_device_add(struct pci_dev *pdev) if (pdev->is_physfn) pdev->no_vf_scan = 1; - pdev->dev.groups = zpci_attr_groups; zpci_map_resources(pdev); for (i = 0; i < PCI_STD_NUM_BARS; i++) { diff --git a/arch/s390/pci/pci_sysfs.c b/arch/s390/pci/pci_sysfs.c index 0f4f1e8fc480..1f81f6ff7b95 100644 --- a/arch/s390/pci/pci_sysfs.c +++ b/arch/s390/pci/pci_sysfs.c @@ -197,7 +197,7 @@ static struct attribute *zpci_ident_attrs[] = { NULL, }; -static struct attribute_group zpci_ident_attr_group = { +const struct attribute_group zpci_ident_attr_group = { .attrs = zpci_ident_attrs, .is_visible = zpci_index_is_visible, }; @@ -223,7 +223,7 @@ static struct attribute *zpci_dev_attrs[] = { NULL, }; -static struct attribute_group zpci_attr_group = { +const struct attribute_group zpci_attr_group = { .attrs = zpci_dev_attrs, .bin_attrs = zpci_bin_attrs, }; @@ -235,14 +235,8 @@ static struct attribute *pfip_attrs[] = { &dev_attr_segment3.attr, NULL, }; -static struct attribute_group pfip_attr_group = { + +const struct attribute_group pfip_attr_group = { .name = "pfip", .attrs = pfip_attrs, }; - -const struct attribute_group *zpci_attr_groups[] = { - &zpci_attr_group, - &pfip_attr_group, - &zpci_ident_attr_group, - NULL, -}; diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c index 40cfa716392f..5d0f4db1cab7 100644 --- a/drivers/pci/pci-sysfs.c +++ b/drivers/pci/pci-sysfs.c @@ -31,6 +31,10 @@ #include #include "pci.h" +#ifndef ARCH_PCI_DEV_GROUPS +#define ARCH_PCI_DEV_GROUPS +#endif + static int sysfs_initialized; /* = 0 */ /* show configuration fields */ @@ -1624,6 +1628,7 @@ const struct attribute_group *pci_dev_groups[] = { &pci_dev_acpi_attr_group, #endif &pci_dev_resource_resize_group, + ARCH_PCI_DEV_GROUPS NULL, }; From 7ff7509fa52397455e04bd44982e9dfbbd19457f Mon Sep 17 00:00:00 2001 From: Philipp Stanner Date: Mon, 29 Jul 2024 11:36:26 +0200 Subject: [PATCH 025/302] PCI: Make pcim_request_region() a public function pcim_request_region() is the managed counterpart of pci_request_region(). It is currently only used internally for PCI. It can be useful for a number of drivers and exporting it is a step towards deprecating more complicated functions. Make pcim_request_region() a public function. Link: https://lore.kernel.org/r/20240729093625.17561-4-pstanner@redhat.com Signed-off-by: Philipp Stanner Signed-off-by: Bjorn Helgaas Tested-by: Hans de Goede Reviewed-by: Hans de Goede Acked-by: Hans de Goede --- drivers/pci/devres.c | 1 + drivers/pci/pci.h | 2 -- include/linux/pci.h | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/pci/devres.c b/drivers/pci/devres.c index 3780a9f9ec00..0127ca58c6e5 100644 --- a/drivers/pci/devres.c +++ b/drivers/pci/devres.c @@ -863,6 +863,7 @@ int pcim_request_region(struct pci_dev *pdev, int bar, const char *name) { return _pcim_request_region(pdev, bar, name, 0); } +EXPORT_SYMBOL(pcim_request_region); /** * pcim_request_region_exclusive - Request a PCI BAR exclusively diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 79c8398f3938..2fe6055a334d 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -887,8 +887,6 @@ static inline pci_power_t mid_pci_get_power_state(struct pci_dev *pdev) #endif int pcim_intx(struct pci_dev *dev, int enable); - -int pcim_request_region(struct pci_dev *pdev, int bar, const char *name); int pcim_request_region_exclusive(struct pci_dev *pdev, int bar, const char *name); void pcim_release_region(struct pci_dev *pdev, int bar); diff --git a/include/linux/pci.h b/include/linux/pci.h index 4cf89a4b4cbc..4246cb790c7b 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -2292,6 +2292,7 @@ static inline void pci_fixup_device(enum pci_fixup_pass pass, void __iomem *pcim_iomap(struct pci_dev *pdev, int bar, unsigned long maxlen); void pcim_iounmap(struct pci_dev *pdev, void __iomem *addr); void __iomem * const *pcim_iomap_table(struct pci_dev *pdev); +int pcim_request_region(struct pci_dev *pdev, int bar, const char *name); int pcim_iomap_regions(struct pci_dev *pdev, int mask, const char *name); int pcim_iomap_regions_request_all(struct pci_dev *pdev, int mask, const char *name); From 89121e5d0857bbf41f1987c3c6019f4f318aec4c Mon Sep 17 00:00:00 2001 From: Philipp Stanner Date: Mon, 29 Jul 2024 11:36:27 +0200 Subject: [PATCH 026/302] drm/vboxvideo: Add PCI region request vboxvideo currently does not reserve its PCI BAR through a region request. Implement the request through the managed function pcim_request_region(). Link: https://lore.kernel.org/r/20240729093625.17561-5-pstanner@redhat.com Signed-off-by: Philipp Stanner Signed-off-by: Bjorn Helgaas Tested-by: Hans de Goede Reviewed-by: Hans de Goede Acked-by: Hans de Goede --- drivers/gpu/drm/vboxvideo/vbox_main.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/gpu/drm/vboxvideo/vbox_main.c b/drivers/gpu/drm/vboxvideo/vbox_main.c index d4ade9325401..7f686a0190e6 100644 --- a/drivers/gpu/drm/vboxvideo/vbox_main.c +++ b/drivers/gpu/drm/vboxvideo/vbox_main.c @@ -114,6 +114,10 @@ int vbox_hw_init(struct vbox_private *vbox) DRM_INFO("VRAM %08x\n", vbox->full_vram_size); + ret = pcim_request_region(pdev, 0, "vboxvideo"); + if (ret) + return ret; + /* Map guest-heap at end of vram */ vbox->guest_heap = pcim_iomap_range(pdev, 0, GUEST_HEAP_OFFSET(vbox), GUEST_HEAP_SIZE); From d140f80f603584d8282817994c0c6241e736bef4 Mon Sep 17 00:00:00 2001 From: Philipp Stanner Date: Wed, 7 Aug 2024 10:30:18 +0200 Subject: [PATCH 027/302] PCI: Deprecate pcim_iomap_regions() in favor of pcim_iomap_region() pcim_iomap_regions() is a complicated function that uses a bit mask to determine the BARs the user wishes to request and ioremap. Almost all users only ever set a single bit in that mask, making that mechanism questionable. pcim_iomap_region() is now available as a more simple replacement. Make pcim_iomap_region() a public function. Mark pcim_iomap_regions() as deprecated. Link: https://lore.kernel.org/r/20240807083018.8734-2-pstanner@redhat.com Signed-off-by: Philipp Stanner Signed-off-by: Bjorn Helgaas --- drivers/pci/devres.c | 8 ++++++-- include/linux/pci.h | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/pci/devres.c b/drivers/pci/devres.c index 0127ca58c6e5..b97589e99fad 100644 --- a/drivers/pci/devres.c +++ b/drivers/pci/devres.c @@ -728,7 +728,7 @@ EXPORT_SYMBOL(pcim_iounmap); * Mapping and region will get automatically released on driver detach. If * desired, release manually only with pcim_iounmap_region(). */ -static void __iomem *pcim_iomap_region(struct pci_dev *pdev, int bar, +void __iomem *pcim_iomap_region(struct pci_dev *pdev, int bar, const char *name) { int ret; @@ -761,6 +761,7 @@ err_region: return IOMEM_ERR_PTR(ret); } +EXPORT_SYMBOL(pcim_iomap_region); /** * pcim_iounmap_region - Unmap and release a PCI BAR @@ -783,7 +784,7 @@ static void pcim_iounmap_region(struct pci_dev *pdev, int bar) } /** - * pcim_iomap_regions - Request and iomap PCI BARs + * pcim_iomap_regions - Request and iomap PCI BARs (DEPRECATED) * @pdev: PCI device to map IO resources for * @mask: Mask of BARs to request and iomap * @name: Name associated with the requests @@ -791,6 +792,9 @@ static void pcim_iounmap_region(struct pci_dev *pdev, int bar) * Returns: 0 on success, negative error code on failure. * * Request and iomap regions specified by @mask. + * + * This function is DEPRECATED. Do not use it in new code. + * Use pcim_iomap_region() instead. */ int pcim_iomap_regions(struct pci_dev *pdev, int mask, const char *name) { diff --git a/include/linux/pci.h b/include/linux/pci.h index 4246cb790c7b..aa9ef7890c50 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -2290,6 +2290,8 @@ static inline void pci_fixup_device(enum pci_fixup_pass pass, #endif void __iomem *pcim_iomap(struct pci_dev *pdev, int bar, unsigned long maxlen); +void __iomem *pcim_iomap_region(struct pci_dev *pdev, int bar, + const char *name); void pcim_iounmap(struct pci_dev *pdev, void __iomem *addr); void __iomem * const *pcim_iomap_table(struct pci_dev *pdev); int pcim_request_region(struct pci_dev *pdev, int bar, const char *name); From 2eb20b96d7696dc354e1b38c511418b56291013c Mon Sep 17 00:00:00 2001 From: Philipp Stanner Date: Wed, 7 Aug 2024 10:30:20 +0200 Subject: [PATCH 028/302] drm/ast: Request PCI BAR with devres ast currently ioremaps two PCI BARs using pcim_iomap(). It does not perform a request on the regions, however, which would make the driver a bit more robust. PCI now offers pcim_iomap_region(), a managed function which both requests and ioremaps a BAR. Replace pcim_iomap() with pcim_iomap_region(). Suggested-by: Thomas Zimmermann Link: https://lore.kernel.org/r/20240807083018.8734-4-pstanner@redhat.com Signed-off-by: Philipp Stanner Signed-off-by: Bjorn Helgaas Reviewed-by: Thomas Zimmermann Acked-by: Dave Airlie --- drivers/gpu/drm/ast/ast_drv.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/ast/ast_drv.c b/drivers/gpu/drm/ast/ast_drv.c index aae019e79bda..1fadaadfbe39 100644 --- a/drivers/gpu/drm/ast/ast_drv.c +++ b/drivers/gpu/drm/ast/ast_drv.c @@ -287,9 +287,9 @@ static int ast_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (ret) return ret; - regs = pcim_iomap(pdev, 1, 0); - if (!regs) - return -EIO; + regs = pcim_iomap_region(pdev, 1, "ast"); + if (IS_ERR(regs)) + return PTR_ERR(regs); if (pdev->revision >= 0x40) { /* @@ -311,9 +311,9 @@ static int ast_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (len < AST_IO_MM_LENGTH) return -EIO; - ioregs = pcim_iomap(pdev, 2, 0); - if (!ioregs) - return -EIO; + ioregs = pcim_iomap_region(pdev, 2, "ast"); + if (IS_ERR(ioregs)) + return PTR_ERR(ioregs); } else { /* * Anything else is best effort. From 3e40aa29d47e231a54640addf6a09c1f64c5b63f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ilpo=20J=C3=A4rvinen?= Date: Thu, 8 Aug 2024 15:17:07 +0300 Subject: [PATCH 029/302] PCI: Wait for Link before restoring Downstream Buses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit __pci_reset_bus() calls pci_bridge_secondary_bus_reset() to perform the reset and also waits for the Secondary Bus to become again accessible. __pci_reset_bus() then calls pci_bus_restore_locked() that restores the PCI devices connected to the bus, and if necessary, recursively restores also the subordinate buses and their devices. The logic in pci_bus_restore_locked() does not take into account that after restoring a device on one level, there might be another Link Downstream that can only start to come up after restore has been performed for its Downstream Port device. That is, the Link may require additional wait until it becomes accessible. Similarly, pci_slot_restore_locked() lacks wait. Amend pci_bus_restore_locked() and pci_slot_restore_locked() to wait for the Secondary Bus before recursively performing the restore of that bus. Fixes: 090a3c5322e9 ("PCI: Add pci_reset_slot() and pci_reset_bus()") Link: https://lore.kernel.org/r/20240808121708.2523-1-ilpo.jarvinen@linux.intel.com Signed-off-by: Ilpo Järvinen Signed-off-by: Bjorn Helgaas --- drivers/pci/pci.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index e3a49f66982d..98c7b732998a 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -5671,8 +5671,10 @@ static void pci_bus_restore_locked(struct pci_bus *bus) list_for_each_entry(dev, &bus->devices, bus_list) { pci_dev_restore(dev); - if (dev->subordinate) + if (dev->subordinate) { + pci_bridge_wait_for_secondary_bus(dev, "bus reset"); pci_bus_restore_locked(dev->subordinate); + } } } @@ -5706,8 +5708,10 @@ static void pci_slot_restore_locked(struct pci_slot *slot) if (!dev->slot || dev->slot != slot) continue; pci_dev_restore(dev); - if (dev->subordinate) + if (dev->subordinate) { + pci_bridge_wait_for_secondary_bus(dev, "slot reset"); pci_bus_restore_locked(dev->subordinate); + } } } From 6188a1c762eb9bbd444f47696eda77a5eae6207a Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Fri, 19 Jul 2024 18:53:26 -0500 Subject: [PATCH 030/302] PCI: keystone: Fix if-statement expression in ks_pcie_quirk() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This code accidentally uses && where || was intended. It potentially results in a NULL dereference. Thus, fix the if-statement expression to use the correct condition. Fixes: 86f271f22bbb ("PCI: keystone: Add workaround for Errata #i2037 (AM65x SR 1.0)") Link: https://lore.kernel.org/linux-pci/1b762a93-e1b2-4af3-8c04-c8843905c279@stanley.mountain Signed-off-by: Dan Carpenter [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Manivannan Sadhasivam Reviewed-by: Siddharth Vadapalli --- drivers/pci/controller/dwc/pci-keystone.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/pci/controller/dwc/pci-keystone.c b/drivers/pci/controller/dwc/pci-keystone.c index 52c6420ae200..95a471d6a586 100644 --- a/drivers/pci/controller/dwc/pci-keystone.c +++ b/drivers/pci/controller/dwc/pci-keystone.c @@ -577,7 +577,7 @@ static void ks_pcie_quirk(struct pci_dev *dev) */ if (pci_match_id(am6_pci_devids, bridge)) { bridge_dev = pci_get_host_bridge_device(dev); - if (!bridge_dev && !bridge_dev->parent) + if (!bridge_dev || !bridge_dev->parent) return; ks_pcie = dev_get_drvdata(bridge_dev->parent); From 4117efd5c9ecd9d1d531f85967df7a394d783cf2 Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Mon, 19 Aug 2024 13:32:44 +0200 Subject: [PATCH 031/302] gfs2: Minor gfs2_glock_cb cleanup In gfs2_glock_cb(), we only need to calculate the glock hold time for inode glocks; the value is unused otherwise. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/glock.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/fs/gfs2/glock.c b/fs/gfs2/glock.c index 12a769077ea0..4a026703a28d 100644 --- a/fs/gfs2/glock.c +++ b/fs/gfs2/glock.c @@ -1885,14 +1885,16 @@ void gfs2_glock_dq_m(unsigned int num_gh, struct gfs2_holder *ghs) void gfs2_glock_cb(struct gfs2_glock *gl, unsigned int state) { unsigned long delay = 0; - unsigned long holdtime; - unsigned long now = jiffies; gfs2_glock_hold(gl); spin_lock(&gl->gl_lockref.lock); - holdtime = gl->gl_tchange + gl->gl_hold_time; if (!list_empty(&gl->gl_holders) && gl->gl_name.ln_type == LM_TYPE_INODE) { + unsigned long now = jiffies; + unsigned long holdtime; + + holdtime = gl->gl_tchange + gl->gl_hold_time; + if (time_before(now, holdtime)) delay = holdtime - now; if (test_bit(GLF_HAVE_REPLY, &gl->gl_flags)) From 6cb9df81a2c462b89d2f9611009ab43ae8717841 Mon Sep 17 00:00:00 2001 From: Julian Sun Date: Tue, 20 Aug 2024 11:31:48 +0800 Subject: [PATCH 032/302] gfs2: fix double destroy_workqueue error When gfs2_fill_super() fails, destroy_workqueue() is called within gfs2_gl_hash_clear(), and the subsequent code path calls destroy_workqueue() on the same work queue again. This issue can be fixed by setting the work queue pointer to NULL after the first destroy_workqueue() call and checking for a NULL pointer before attempting to destroy the work queue again. Reported-by: syzbot+d34c2a269ed512c531b0@syzkaller.appspotmail.com Closes: https://syzkaller.appspot.com/bug?extid=d34c2a269ed512c531b0 Fixes: 30e388d57367 ("gfs2: Switch to a per-filesystem glock workqueue") Cc: stable@vger.kernel.org Signed-off-by: Julian Sun Signed-off-by: Andreas Gruenbacher --- fs/gfs2/glock.c | 1 + fs/gfs2/ops_fstype.c | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/fs/gfs2/glock.c b/fs/gfs2/glock.c index 4a026703a28d..269c3bc7fced 100644 --- a/fs/gfs2/glock.c +++ b/fs/gfs2/glock.c @@ -2251,6 +2251,7 @@ void gfs2_gl_hash_clear(struct gfs2_sbd *sdp) gfs2_free_dead_glocks(sdp); glock_hash_walk(dump_glock_func, sdp); destroy_workqueue(sdp->sd_glock_wq); + sdp->sd_glock_wq = NULL; } static const char *state2str(unsigned state) diff --git a/fs/gfs2/ops_fstype.c b/fs/gfs2/ops_fstype.c index ff1f3e3dc65c..e83d293c3614 100644 --- a/fs/gfs2/ops_fstype.c +++ b/fs/gfs2/ops_fstype.c @@ -1307,7 +1307,8 @@ fail_debug: fail_delete_wq: destroy_workqueue(sdp->sd_delete_wq); fail_glock_wq: - destroy_workqueue(sdp->sd_glock_wq); + if (sdp->sd_glock_wq) + destroy_workqueue(sdp->sd_glock_wq); fail_free: free_sbd(sdp); sb->s_fs_info = NULL; From dbc3171194403d0d40e4bdeae666f6e76e428b53 Mon Sep 17 00:00:00 2001 From: Samasth Norway Ananda Date: Mon, 12 Aug 2024 13:26:59 -0700 Subject: [PATCH 033/302] x86/PCI: Check pcie_find_root_port() return for NULL If pcie_find_root_port() is unable to locate a Root Port, it will return NULL. Check the pointer for NULL before dereferencing it. This particular case is in a quirk for devices that are always below a Root Port, so this won't avoid a problem and doesn't need to be backported, but check as a matter of style and to prevent copy/paste mistakes. Link: https://lore.kernel.org/r/20240812202659.1649121-1-samasth.norway.ananda@oracle.com Signed-off-by: Samasth Norway Ananda [bhelgaas: drop Fixes: and explain why there's no problem in this case] Signed-off-by: Bjorn Helgaas Reviewed-by: Mario Limonciello --- arch/x86/pci/fixup.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/pci/fixup.c b/arch/x86/pci/fixup.c index b33afb240601..98a9bb92d75c 100644 --- a/arch/x86/pci/fixup.c +++ b/arch/x86/pci/fixup.c @@ -980,7 +980,7 @@ static void amd_rp_pme_suspend(struct pci_dev *dev) return; rp = pcie_find_root_port(dev); - if (!rp->pm_cap) + if (!rp || !rp->pm_cap) return; rp->pme_support &= ~((PCI_PM_CAP_PME_D3hot|PCI_PM_CAP_PME_D3cold) >> @@ -994,7 +994,7 @@ static void amd_rp_pme_resume(struct pci_dev *dev) u16 pmc; rp = pcie_find_root_port(dev); - if (!rp->pm_cap) + if (!rp || !rp->pm_cap) return; pci_read_config_word(rp, rp->pm_cap + PCI_PM_PMC, &pmc); From 0199d2f2bd8cd97b310f7ed82a067247d7456029 Mon Sep 17 00:00:00 2001 From: Sean Anderson Date: Fri, 31 May 2024 12:13:32 -0400 Subject: [PATCH 034/302] PCI: xilinx-nwl: Fix off-by-one in INTx IRQ handler MSGF_LEG_MASK is laid out with INTA in bit 0, INTB in bit 1, INTC in bit 2, and INTD in bit 3. Hardware IRQ numbers start at 0, and we register PCI_NUM_INTX IRQs. So to enable INTA (aka hwirq 0) we should set bit 0. Remove the subtraction of one. This bug would cause INTx interrupts not to be delivered, as enabling INTB would actually enable INTA, and enabling INTA wouldn't enable anything at all. It is likely that this got overlooked for so long since most PCIe hardware uses MSIs. This fixes the following UBSAN error: UBSAN: shift-out-of-bounds in ../drivers/pci/controller/pcie-xilinx-nwl.c:389:11 shift exponent 18446744073709551615 is too large for 32-bit type 'int' CPU: 1 PID: 61 Comm: kworker/u10:1 Not tainted 6.6.20+ #268 Hardware name: xlnx,zynqmp (DT) Workqueue: events_unbound deferred_probe_work_func Call trace: dump_backtrace (arch/arm64/kernel/stacktrace.c:235) show_stack (arch/arm64/kernel/stacktrace.c:242) dump_stack_lvl (lib/dump_stack.c:107) dump_stack (lib/dump_stack.c:114) __ubsan_handle_shift_out_of_bounds (lib/ubsan.c:218 lib/ubsan.c:387) nwl_unmask_leg_irq (drivers/pci/controller/pcie-xilinx-nwl.c:389 (discriminator 1)) irq_enable (kernel/irq/internals.h:234 kernel/irq/chip.c:170 kernel/irq/chip.c:439 kernel/irq/chip.c:432 kernel/irq/chip.c:345) __irq_startup (kernel/irq/internals.h:239 kernel/irq/chip.c:180 kernel/irq/chip.c:250) irq_startup (kernel/irq/chip.c:270) __setup_irq (kernel/irq/manage.c:1800) request_threaded_irq (kernel/irq/manage.c:2206) pcie_pme_probe (include/linux/interrupt.h:168 drivers/pci/pcie/pme.c:348) Fixes: 9a181e1093af ("PCI: xilinx-nwl: Modify IRQ chip for legacy interrupts") Link: https://lore.kernel.org/r/20240531161337.864994-3-sean.anderson@linux.dev Signed-off-by: Sean Anderson Signed-off-by: Bjorn Helgaas Cc: stable@vger.kernel.org --- drivers/pci/controller/pcie-xilinx-nwl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/pci/controller/pcie-xilinx-nwl.c b/drivers/pci/controller/pcie-xilinx-nwl.c index 0408f4d612b5..437927e3bcca 100644 --- a/drivers/pci/controller/pcie-xilinx-nwl.c +++ b/drivers/pci/controller/pcie-xilinx-nwl.c @@ -371,7 +371,7 @@ static void nwl_mask_intx_irq(struct irq_data *data) u32 mask; u32 val; - mask = 1 << (data->hwirq - 1); + mask = 1 << data->hwirq; raw_spin_lock_irqsave(&pcie->leg_mask_lock, flags); val = nwl_bridge_readl(pcie, MSGF_LEG_MASK); nwl_bridge_writel(pcie, (val & (~mask)), MSGF_LEG_MASK); @@ -385,7 +385,7 @@ static void nwl_unmask_intx_irq(struct irq_data *data) u32 mask; u32 val; - mask = 1 << (data->hwirq - 1); + mask = 1 << data->hwirq; raw_spin_lock_irqsave(&pcie->leg_mask_lock, flags); val = nwl_bridge_readl(pcie, MSGF_LEG_MASK); nwl_bridge_writel(pcie, (val | mask), MSGF_LEG_MASK); From a437027ae1730b8dc379c75fa0dd7d3036917400 Mon Sep 17 00:00:00 2001 From: Sean Anderson Date: Fri, 31 May 2024 12:13:33 -0400 Subject: [PATCH 035/302] PCI: xilinx-nwl: Fix register misspelling MSIC -> MISC Fixes: c2a7ff18edcd ("PCI: xilinx-nwl: Expand error logging") Link: https://lore.kernel.org/r/20240531161337.864994-4-sean.anderson@linux.dev Signed-off-by: Sean Anderson Signed-off-by: Bjorn Helgaas --- drivers/pci/controller/pcie-xilinx-nwl.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/pci/controller/pcie-xilinx-nwl.c b/drivers/pci/controller/pcie-xilinx-nwl.c index 437927e3bcca..ce881baac6d8 100644 --- a/drivers/pci/controller/pcie-xilinx-nwl.c +++ b/drivers/pci/controller/pcie-xilinx-nwl.c @@ -80,8 +80,8 @@ #define MSGF_MISC_SR_NON_FATAL_DEV BIT(22) #define MSGF_MISC_SR_FATAL_DEV BIT(23) #define MSGF_MISC_SR_LINK_DOWN BIT(24) -#define MSGF_MSIC_SR_LINK_AUTO_BWIDTH BIT(25) -#define MSGF_MSIC_SR_LINK_BWIDTH BIT(26) +#define MSGF_MISC_SR_LINK_AUTO_BWIDTH BIT(25) +#define MSGF_MISC_SR_LINK_BWIDTH BIT(26) #define MSGF_MISC_SR_MASKALL (MSGF_MISC_SR_RXMSG_AVAIL | \ MSGF_MISC_SR_RXMSG_OVER | \ @@ -96,8 +96,8 @@ MSGF_MISC_SR_NON_FATAL_DEV | \ MSGF_MISC_SR_FATAL_DEV | \ MSGF_MISC_SR_LINK_DOWN | \ - MSGF_MSIC_SR_LINK_AUTO_BWIDTH | \ - MSGF_MSIC_SR_LINK_BWIDTH) + MSGF_MISC_SR_LINK_AUTO_BWIDTH | \ + MSGF_MISC_SR_LINK_BWIDTH) /* Legacy interrupt status mask bits */ #define MSGF_LEG_SR_INTA BIT(0) @@ -299,10 +299,10 @@ static irqreturn_t nwl_pcie_misc_handler(int irq, void *data) if (misc_stat & MSGF_MISC_SR_FATAL_DEV) dev_err(dev, "Fatal Error Detected\n"); - if (misc_stat & MSGF_MSIC_SR_LINK_AUTO_BWIDTH) + if (misc_stat & MSGF_MISC_SR_LINK_AUTO_BWIDTH) dev_info(dev, "Link Autonomous Bandwidth Management Status bit set\n"); - if (misc_stat & MSGF_MSIC_SR_LINK_BWIDTH) + if (misc_stat & MSGF_MISC_SR_LINK_BWIDTH) dev_info(dev, "Link Bandwidth Management Status bit set\n"); /* Clear misc interrupt status */ From 78457cae24cb543650bec048927bfb4c164d060f Mon Sep 17 00:00:00 2001 From: Sean Anderson Date: Fri, 31 May 2024 12:13:34 -0400 Subject: [PATCH 036/302] PCI: xilinx-nwl: Rate-limit misc interrupt messages The conditions logged by the misc interrupt can occur repeatedly and continuously. Avoid rendering the console unusable by rate-limiting these messages. Link: https://lore.kernel.org/r/20240531161337.864994-5-sean.anderson@linux.dev Signed-off-by: Sean Anderson Signed-off-by: Bjorn Helgaas --- drivers/pci/controller/pcie-xilinx-nwl.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/drivers/pci/controller/pcie-xilinx-nwl.c b/drivers/pci/controller/pcie-xilinx-nwl.c index ce881baac6d8..c0a60cebdb2e 100644 --- a/drivers/pci/controller/pcie-xilinx-nwl.c +++ b/drivers/pci/controller/pcie-xilinx-nwl.c @@ -267,37 +267,37 @@ static irqreturn_t nwl_pcie_misc_handler(int irq, void *data) return IRQ_NONE; if (misc_stat & MSGF_MISC_SR_RXMSG_OVER) - dev_err(dev, "Received Message FIFO Overflow\n"); + dev_err_ratelimited(dev, "Received Message FIFO Overflow\n"); if (misc_stat & MSGF_MISC_SR_SLAVE_ERR) - dev_err(dev, "Slave error\n"); + dev_err_ratelimited(dev, "Slave error\n"); if (misc_stat & MSGF_MISC_SR_MASTER_ERR) - dev_err(dev, "Master error\n"); + dev_err_ratelimited(dev, "Master error\n"); if (misc_stat & MSGF_MISC_SR_I_ADDR_ERR) - dev_err(dev, "In Misc Ingress address translation error\n"); + dev_err_ratelimited(dev, "In Misc Ingress address translation error\n"); if (misc_stat & MSGF_MISC_SR_E_ADDR_ERR) - dev_err(dev, "In Misc Egress address translation error\n"); + dev_err_ratelimited(dev, "In Misc Egress address translation error\n"); if (misc_stat & MSGF_MISC_SR_FATAL_AER) - dev_err(dev, "Fatal Error in AER Capability\n"); + dev_err_ratelimited(dev, "Fatal Error in AER Capability\n"); if (misc_stat & MSGF_MISC_SR_NON_FATAL_AER) - dev_err(dev, "Non-Fatal Error in AER Capability\n"); + dev_err_ratelimited(dev, "Non-Fatal Error in AER Capability\n"); if (misc_stat & MSGF_MISC_SR_CORR_AER) - dev_err(dev, "Correctable Error in AER Capability\n"); + dev_err_ratelimited(dev, "Correctable Error in AER Capability\n"); if (misc_stat & MSGF_MISC_SR_UR_DETECT) - dev_err(dev, "Unsupported request Detected\n"); + dev_err_ratelimited(dev, "Unsupported request Detected\n"); if (misc_stat & MSGF_MISC_SR_NON_FATAL_DEV) - dev_err(dev, "Non-Fatal Error Detected\n"); + dev_err_ratelimited(dev, "Non-Fatal Error Detected\n"); if (misc_stat & MSGF_MISC_SR_FATAL_DEV) - dev_err(dev, "Fatal Error Detected\n"); + dev_err_ratelimited(dev, "Fatal Error Detected\n"); if (misc_stat & MSGF_MISC_SR_LINK_AUTO_BWIDTH) dev_info(dev, "Link Autonomous Bandwidth Management Status bit set\n"); From cfd67903977b13f63340a4eb5a1cc890994f2c62 Mon Sep 17 00:00:00 2001 From: Sean Anderson Date: Fri, 31 May 2024 12:13:35 -0400 Subject: [PATCH 037/302] PCI: xilinx-nwl: Clean up clock on probe failure/removal Make sure we turn off the clock on probe failure and device removal. Fixes: de0a01f52966 ("PCI: xilinx-nwl: Enable the clock through CCF") Link: https://lore.kernel.org/r/20240531161337.864994-6-sean.anderson@linux.dev Signed-off-by: Sean Anderson Signed-off-by: Bjorn Helgaas --- drivers/pci/controller/pcie-xilinx-nwl.c | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/drivers/pci/controller/pcie-xilinx-nwl.c b/drivers/pci/controller/pcie-xilinx-nwl.c index c0a60cebdb2e..e85158dc4e6c 100644 --- a/drivers/pci/controller/pcie-xilinx-nwl.c +++ b/drivers/pci/controller/pcie-xilinx-nwl.c @@ -779,6 +779,7 @@ static int nwl_pcie_probe(struct platform_device *pdev) return -ENODEV; pcie = pci_host_bridge_priv(bridge); + platform_set_drvdata(pdev, pcie); pcie->dev = dev; @@ -801,13 +802,13 @@ static int nwl_pcie_probe(struct platform_device *pdev) err = nwl_pcie_bridge_init(pcie); if (err) { dev_err(dev, "HW Initialization failed\n"); - return err; + goto err_clk; } err = nwl_pcie_init_irq_domain(pcie); if (err) { dev_err(dev, "Failed creating IRQ Domain\n"); - return err; + goto err_clk; } bridge->sysdata = pcie; @@ -817,11 +818,24 @@ static int nwl_pcie_probe(struct platform_device *pdev) err = nwl_pcie_enable_msi(pcie); if (err < 0) { dev_err(dev, "failed to enable MSI support: %d\n", err); - return err; + goto err_clk; } } - return pci_host_probe(bridge); + err = pci_host_probe(bridge); + if (!err) + return 0; + +err_clk: + clk_disable_unprepare(pcie->clk); + return err; +} + +static void nwl_pcie_remove(struct platform_device *pdev) +{ + struct nwl_pcie *pcie = platform_get_drvdata(pdev); + + clk_disable_unprepare(pcie->clk); } static struct platform_driver nwl_pcie_driver = { @@ -831,5 +845,6 @@ static struct platform_driver nwl_pcie_driver = { .of_match_table = nwl_pcie_of_match, }, .probe = nwl_pcie_probe, + .remove_new = nwl_pcie_remove, }; builtin_platform_driver(nwl_pcie_driver); From 3e47bcc9b77df6c00bfa1e464a8f68b632f811f5 Mon Sep 17 00:00:00 2001 From: Sean Anderson Date: Fri, 31 May 2024 12:13:31 -0400 Subject: [PATCH 038/302] dt-bindings: pci: xilinx-nwl: Add phys property Add phys properties so Linux can power-on/configure the GTR transceivers (xlnx,zynqmp-psgtr-v1.1). Link: https://lore.kernel.org/r/20240531161337.864994-2-sean.anderson@linux.dev Signed-off-by: Sean Anderson Signed-off-by: Bjorn Helgaas Reviewed-by: Rob Herring (Arm) --- Documentation/devicetree/bindings/pci/xlnx,nwl-pcie.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Documentation/devicetree/bindings/pci/xlnx,nwl-pcie.yaml b/Documentation/devicetree/bindings/pci/xlnx,nwl-pcie.yaml index 9cad860c51a3..9de3c09efb6e 100644 --- a/Documentation/devicetree/bindings/pci/xlnx,nwl-pcie.yaml +++ b/Documentation/devicetree/bindings/pci/xlnx,nwl-pcie.yaml @@ -61,6 +61,11 @@ properties: interrupt-map: maxItems: 4 + phys: + minItems: 1 + maxItems: 4 + description: One phy per logical lane, in order + power-domains: maxItems: 1 @@ -110,6 +115,7 @@ examples: - | #include #include + #include #include soc { #address-cells = <2>; @@ -138,6 +144,7 @@ examples: <0x0 0x0 0x0 0x3 &pcie_intc 0x3>, <0x0 0x0 0x0 0x4 &pcie_intc 0x4>; msi-parent = <&nwl_pcie>; + phys = <&psgtr 0 PHY_TYPE_PCIE 0 0>; power-domains = <&zynqmp_firmware PD_PCIE>; iommus = <&smmu 0x4d0>; pcie_intc: legacy-interrupt-controller { From 308a40fb8fd9cfb1aa7d44f79980dfaef1a6c72f Mon Sep 17 00:00:00 2001 From: Sean Anderson Date: Fri, 31 May 2024 12:13:36 -0400 Subject: [PATCH 039/302] PCI: xilinx-nwl: Add PHY support Add support for enabling/disabling PCIe PHYs. We can't really do anything about failures in the disable/remove path, so just print an error. Link: https://lore.kernel.org/r/20240531161337.864994-7-sean.anderson@linux.dev Signed-off-by: Sean Anderson Signed-off-by: Bjorn Helgaas --- drivers/pci/controller/pcie-xilinx-nwl.c | 84 +++++++++++++++++++++++- 1 file changed, 81 insertions(+), 3 deletions(-) diff --git a/drivers/pci/controller/pcie-xilinx-nwl.c b/drivers/pci/controller/pcie-xilinx-nwl.c index e85158dc4e6c..d17630357999 100644 --- a/drivers/pci/controller/pcie-xilinx-nwl.c +++ b/drivers/pci/controller/pcie-xilinx-nwl.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -157,6 +158,7 @@ struct nwl_pcie { void __iomem *breg_base; void __iomem *pcireg_base; void __iomem *ecam_base; + struct phy *phy[4]; phys_addr_t phys_breg_base; /* Physical Bridge Register Base */ phys_addr_t phys_pcie_reg_base; /* Physical PCIe Controller Base */ phys_addr_t phys_ecam_base; /* Physical Configuration Base */ @@ -521,6 +523,60 @@ static int nwl_pcie_init_msi_irq_domain(struct nwl_pcie *pcie) return 0; } +static void nwl_pcie_phy_power_off(struct nwl_pcie *pcie, int i) +{ + int err = phy_power_off(pcie->phy[i]); + + if (err) + dev_err(pcie->dev, "could not power off phy %d (err=%d)\n", i, + err); +} + +static void nwl_pcie_phy_exit(struct nwl_pcie *pcie, int i) +{ + int err = phy_exit(pcie->phy[i]); + + if (err) + dev_err(pcie->dev, "could not exit phy %d (err=%d)\n", i, err); +} + +static int nwl_pcie_phy_enable(struct nwl_pcie *pcie) +{ + int i, ret; + + for (i = 0; i < ARRAY_SIZE(pcie->phy); i++) { + ret = phy_init(pcie->phy[i]); + if (ret) + goto err; + + ret = phy_power_on(pcie->phy[i]); + if (ret) { + nwl_pcie_phy_exit(pcie, i); + goto err; + } + } + + return 0; + +err: + while (i--) { + nwl_pcie_phy_power_off(pcie, i); + nwl_pcie_phy_exit(pcie, i); + } + + return ret; +} + +static void nwl_pcie_phy_disable(struct nwl_pcie *pcie) +{ + int i; + + for (i = ARRAY_SIZE(pcie->phy); i--;) { + nwl_pcie_phy_power_off(pcie, i); + nwl_pcie_phy_exit(pcie, i); + } +} + static int nwl_pcie_init_irq_domain(struct nwl_pcie *pcie) { struct device *dev = pcie->dev; @@ -732,6 +788,7 @@ static int nwl_pcie_parse_dt(struct nwl_pcie *pcie, { struct device *dev = pcie->dev; struct resource *res; + int i; res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "breg"); pcie->breg_base = devm_ioremap_resource(dev, res); @@ -759,6 +816,18 @@ static int nwl_pcie_parse_dt(struct nwl_pcie *pcie, irq_set_chained_handler_and_data(pcie->irq_intx, nwl_pcie_leg_handler, pcie); + + for (i = 0; i < ARRAY_SIZE(pcie->phy); i++) { + pcie->phy[i] = devm_of_phy_get_by_index(dev, dev->of_node, i); + if (PTR_ERR(pcie->phy[i]) == -ENODEV) { + pcie->phy[i] = NULL; + break; + } + + if (IS_ERR(pcie->phy[i])) + return PTR_ERR(pcie->phy[i]); + } + return 0; } @@ -799,16 +868,22 @@ static int nwl_pcie_probe(struct platform_device *pdev) return err; } + err = nwl_pcie_phy_enable(pcie); + if (err) { + dev_err(dev, "could not enable PHYs\n"); + goto err_clk; + } + err = nwl_pcie_bridge_init(pcie); if (err) { dev_err(dev, "HW Initialization failed\n"); - goto err_clk; + goto err_phy; } err = nwl_pcie_init_irq_domain(pcie); if (err) { dev_err(dev, "Failed creating IRQ Domain\n"); - goto err_clk; + goto err_phy; } bridge->sysdata = pcie; @@ -818,7 +893,7 @@ static int nwl_pcie_probe(struct platform_device *pdev) err = nwl_pcie_enable_msi(pcie); if (err < 0) { dev_err(dev, "failed to enable MSI support: %d\n", err); - goto err_clk; + goto err_phy; } } @@ -826,6 +901,8 @@ static int nwl_pcie_probe(struct platform_device *pdev) if (!err) return 0; +err_phy: + nwl_pcie_phy_disable(pcie); err_clk: clk_disable_unprepare(pcie->clk); return err; @@ -835,6 +912,7 @@ static void nwl_pcie_remove(struct platform_device *pdev) { struct nwl_pcie *pcie = platform_get_drvdata(pdev); + nwl_pcie_phy_disable(pcie); clk_disable_unprepare(pcie->clk); } From 0da59840f10141988e949d8519ed9182991caf17 Mon Sep 17 00:00:00 2001 From: Konrad Dybcio Date: Tue, 13 Aug 2024 21:12:00 +0200 Subject: [PATCH 040/302] PCI/pwrctl: Add WCN6855 support Add support for ATH11K inside the WCN6855 package to the power sequencing PCI power control driver. Link: https://lore.kernel.org/r/20240813191201.155123-1-brgl@bgdev.pl [Bartosz: split Konrad's bigger patch, write the commit message] Co-developed-by: Bartosz Golaszewski Signed-off-by: Konrad Dybcio Signed-off-by: Bartosz Golaszewski Signed-off-by: Bjorn Helgaas Reviewed-by: Konrad Dybcio --- drivers/pci/pwrctl/pci-pwrctl-pwrseq.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/pci/pwrctl/pci-pwrctl-pwrseq.c b/drivers/pci/pwrctl/pci-pwrctl-pwrseq.c index c7a113a76c0c..9a22f4528dd9 100644 --- a/drivers/pci/pwrctl/pci-pwrctl-pwrseq.c +++ b/drivers/pci/pwrctl/pci-pwrctl-pwrseq.c @@ -66,6 +66,11 @@ static const struct of_device_id pci_pwrctl_pwrseq_of_match[] = { .compatible = "pci17cb,1101", .data = "wlan", }, + { + /* ATH11K in WCN6855 package. */ + .compatible = "pci17cb,1103", + .data = "wlan", + }, { /* ATH12K in WCN7850 package. */ .compatible = "pci17cb,1107", From 8a48281cfa7f380707d42b649acda3ea36348697 Mon Sep 17 00:00:00 2001 From: Kunwu Chan Date: Fri, 23 Aug 2024 15:42:01 +0800 Subject: [PATCH 041/302] PCI: Make pci_bus_type constant Since commit d492cc2573a0 ("driver core: device.h: make struct bus_type a const *"), the driver core can properly handle constant struct bus_type, move the pci_bus_type variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Link: https://lore.kernel.org/r/20240823074202.139265-1-kunwu.chan@linux.dev Suggested-by: Greg Kroah-Hartman Signed-off-by: Kunwu Chan Signed-off-by: Bjorn Helgaas Cc: Greg Kroah-Hartman --- drivers/pci/pci-driver.c | 2 +- include/linux/pci.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/pci/pci-driver.c b/drivers/pci/pci-driver.c index f412ef73a6e4..35270172c833 100644 --- a/drivers/pci/pci-driver.c +++ b/drivers/pci/pci-driver.c @@ -1670,7 +1670,7 @@ static void pci_dma_cleanup(struct device *dev) iommu_device_unuse_default_domain(dev); } -struct bus_type pci_bus_type = { +const struct bus_type pci_bus_type = { .name = "pci", .match = pci_bus_match, .uevent = pci_uevent, diff --git a/include/linux/pci.h b/include/linux/pci.h index 4cf89a4b4cbc..197292b336a5 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -1098,7 +1098,7 @@ enum pcie_bus_config_types { extern enum pcie_bus_config_types pcie_bus_config; -extern struct bus_type pci_bus_type; +extern const struct bus_type pci_bus_type; /* Do NOT directly access these two variables, unless you are arch-specific PCI * code, or PCI core code. */ From 9246b487ab3c3b5993aae7552b7a4c541cc14a49 Mon Sep 17 00:00:00 2001 From: WangYuli Date: Fri, 23 Aug 2024 17:57:08 +0800 Subject: [PATCH 042/302] PCI: Add function 0 DMA alias quirk for Glenfly Arise chip Add DMA support for audio function of Glenfly Arise chip, which uses Requester ID of function 0. Link: https://lore.kernel.org/r/CA2BBD087345B6D1+20240823095708.3237375-1-wangyuli@uniontech.com Signed-off-by: SiyuLi Signed-off-by: WangYuli [bhelgaas: lower-case hex to match local code, drop unused Device IDs] Signed-off-by: Bjorn Helgaas Reviewed-by: Takashi Iwai --- drivers/pci/quirks.c | 4 ++++ include/linux/pci_ids.h | 2 ++ sound/pci/hda/hda_intel.c | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index a2ce4e08edf5..cc6c82c3bd3d 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -4246,6 +4246,10 @@ static void quirk_dma_func0_alias(struct pci_dev *dev) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_RICOH, 0xe832, quirk_dma_func0_alias); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_RICOH, 0xe476, quirk_dma_func0_alias); +/* Some Glenfly chips use function 0 as the PCIe Requester ID for DMA */ +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_GLENFLY, 0x3d40, quirk_dma_func0_alias); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_GLENFLY, 0x3d41, quirk_dma_func0_alias); + static void quirk_dma_func1_alias(struct pci_dev *dev) { if (PCI_FUNC(dev->devfn) != 1) diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h index e388c8b1cbc2..2c94d4004dd5 100644 --- a/include/linux/pci_ids.h +++ b/include/linux/pci_ids.h @@ -2661,6 +2661,8 @@ #define PCI_DEVICE_ID_DCI_PCCOM8 0x0002 #define PCI_DEVICE_ID_DCI_PCCOM2 0x0004 +#define PCI_VENDOR_ID_GLENFLY 0x6766 + #define PCI_VENDOR_ID_INTEL 0x8086 #define PCI_DEVICE_ID_INTEL_EESSC 0x0008 #define PCI_DEVICE_ID_INTEL_HDA_CML_LP 0x02c8 diff --git a/sound/pci/hda/hda_intel.c b/sound/pci/hda/hda_intel.c index b33602e64d17..e8958a464647 100644 --- a/sound/pci/hda/hda_intel.c +++ b/sound/pci/hda/hda_intel.c @@ -2671,7 +2671,7 @@ static const struct pci_device_id azx_ids[] = { .driver_data = AZX_DRIVER_ATIHDMI_NS | AZX_DCAPS_PRESET_ATI_HDMI_NS | AZX_DCAPS_PM_RUNTIME }, /* GLENFLY */ - { PCI_DEVICE(0x6766, PCI_ANY_ID), + { PCI_DEVICE(PCI_VENDOR_ID_GLENFLY, PCI_ANY_ID), .class = PCI_CLASS_MULTIMEDIA_HD_AUDIO << 8, .class_mask = 0xffffff, .driver_data = AZX_DRIVER_GFHDMI | AZX_DCAPS_POSFIX_LPIB | From 47ecb443a91ad3d82a79807bec7d258cad8eb86a Mon Sep 17 00:00:00 2001 From: Yue Haibing Date: Wed, 28 Aug 2024 16:21:41 -0500 Subject: [PATCH 043/302] PCI/PM: Remove __pci_pme_wakeup() unused declarations Commit b67ea76172d4 ("PCI / ACPI / PM: Platform support for PCI PME wake-up") declared but never implemented __pci_pme_wakeup(). Remove the unused declaration. Link: https://lore.kernel.org/r/20240824100331.586036-1-yuehaibing@huawei.com Signed-off-by: Yue Haibing [bhelgaas: split per feature] Signed-off-by: Bjorn Helgaas --- drivers/pci/pci.h | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 79c8398f3938..c69383b7a5bc 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -124,7 +124,6 @@ void pcie_clear_device_status(struct pci_dev *dev); void pcie_clear_root_pme_status(struct pci_dev *dev); bool pci_check_pme_status(struct pci_dev *dev); void pci_pme_wakeup_bus(struct pci_bus *bus); -int __pci_pme_wakeup(struct pci_dev *dev, void *ign); void pci_pme_restore(struct pci_dev *dev); bool pci_dev_need_resume(struct pci_dev *dev); void pci_dev_adjust_pme(struct pci_dev *dev); From 2ac11a2fdcc3aa210d76b65c61c6758f44ac6c54 Mon Sep 17 00:00:00 2001 From: Yue Haibing Date: Sat, 24 Aug 2024 18:03:31 +0800 Subject: [PATCH 044/302] PCI/VPD: Remove pci_vpd_release() unused declarations Commit fd00faa375fb ("PCI/VPD: Embed struct pci_vpd in struct pci_dev") removed pci_vpd_release() but left the declaration. Remove the unused declaration. Link: https://lore.kernel.org/r/20240824100331.586036-1-yuehaibing@huawei.com Signed-off-by: Yue Haibing [bhelgaas: split per feature] Signed-off-by: Bjorn Helgaas --- drivers/pci/pci.h | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index c69383b7a5bc..6dbd012eb681 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -168,7 +168,6 @@ static inline bool pcie_downstream_port(const struct pci_dev *dev) } void pci_vpd_init(struct pci_dev *dev); -void pci_vpd_release(struct pci_dev *dev); extern const struct attribute_group pci_dev_vpd_attr_group; /* PCI Virtual Channel */ From 0b93267adb34f39ca5b1574cf3766ee5fe3d44b6 Mon Sep 17 00:00:00 2001 From: Sean Anderson Date: Fri, 31 May 2024 12:13:37 -0400 Subject: [PATCH 045/302] arm64: zynqmp: Add PCIe phys property for ZCU102 Add PCIe phy bindings for the ZCU102. Link: https://lore.kernel.org/r/20240531161337.864994-8-sean.anderson@linux.dev Tested-by: Thippeswamy Havalige Signed-off-by: Sean Anderson Signed-off-by: Bjorn Helgaas Acked-by: Michal Simek --- arch/arm64/boot/dts/xilinx/zynqmp-zcu102-revA.dts | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zcu102-revA.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zcu102-revA.dts index ad8f23a0ec67..d2175f3dd099 100644 --- a/arch/arm64/boot/dts/xilinx/zynqmp-zcu102-revA.dts +++ b/arch/arm64/boot/dts/xilinx/zynqmp-zcu102-revA.dts @@ -941,6 +941,7 @@ &pcie { status = "okay"; + phys = <&psgtr 0 PHY_TYPE_PCIE 0 0>; }; &psgtr { From 899d54826110af3a896a614c9d065ba6acf1e8b3 Mon Sep 17 00:00:00 2001 From: Thippeswamy Havalige Date: Sun, 11 Aug 2024 07:53:44 +0530 Subject: [PATCH 046/302] dt-bindings: PCI: xilinx-xdma: Add schemas for Xilinx QDMA PCIe Root Port Bridge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add YAML devicetree schemas for Xilinx QDMA Soft IP PCIe Root Port Bridge version 3.0. Link: https://lore.kernel.org/linux-pci/20240811022345.1178203-2-thippesw@amd.com Signed-off-by: Thippeswamy Havalige Signed-off-by: Krzysztof Wilczyński Acked-by: Conor Dooley --- .../bindings/pci/xlnx,xdma-host.yaml | 36 +++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/pci/xlnx,xdma-host.yaml b/Documentation/devicetree/bindings/pci/xlnx,xdma-host.yaml index 2f59b3a73dd2..f1efd919c351 100644 --- a/Documentation/devicetree/bindings/pci/xlnx,xdma-host.yaml +++ b/Documentation/devicetree/bindings/pci/xlnx,xdma-host.yaml @@ -14,10 +14,21 @@ allOf: properties: compatible: - const: xlnx,xdma-host-3.00 + enum: + - xlnx,xdma-host-3.00 + - xlnx,qdma-host-3.00 reg: - maxItems: 1 + items: + - description: configuration region and XDMA bridge register. + - description: QDMA bridge register. + minItems: 1 + + reg-names: + items: + - const: cfg + - const: breg + minItems: 1 ranges: maxItems: 2 @@ -76,6 +87,27 @@ required: - "#interrupt-cells" - interrupt-controller +if: + properties: + compatible: + contains: + enum: + - xlnx,qdma-host-3.00 +then: + properties: + reg: + minItems: 2 + reg-names: + minItems: 2 + required: + - reg-names +else: + properties: + reg: + maxItems: 1 + reg-names: + maxItems: 1 + unevaluatedProperties: false examples: From 6ac721795d7375e48bede6bb4965c0e46133c704 Mon Sep 17 00:00:00 2001 From: Thippeswamy Havalige Date: Sun, 11 Aug 2024 07:53:45 +0530 Subject: [PATCH 047/302] PCI: xilinx-xdma: Add Xilinx QDMA Root Port driver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add support for Xilinx QDMA Soft IP core as Root Port. The Versal Prime devices support QDMA soft IP module in programmable logic. The integrated QDMA Soft IP block has integrated bridge function that can act as PCIe Root Port. Link: https://lore.kernel.org/linux-pci/20240811022345.1178203-3-thippesw@amd.com Signed-off-by: Thippeswamy Havalige [kwilczynski: removed unused header] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Manivannan Sadhasivam --- drivers/pci/controller/pcie-xilinx-dma-pl.c | 53 ++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/drivers/pci/controller/pcie-xilinx-dma-pl.c b/drivers/pci/controller/pcie-xilinx-dma-pl.c index 5be5dfd8398f..58b3578892d8 100644 --- a/drivers/pci/controller/pcie-xilinx-dma-pl.c +++ b/drivers/pci/controller/pcie-xilinx-dma-pl.c @@ -71,10 +71,24 @@ /* Phy Status/Control Register definitions */ #define XILINX_PCIE_DMA_REG_PSCR_LNKUP BIT(11) +#define QDMA_BRIDGE_BASE_OFF 0xcd8 /* Number of MSI IRQs */ #define XILINX_NUM_MSI_IRQS 64 +enum xilinx_pl_dma_version { + XDMA, + QDMA, +}; + +/** + * struct xilinx_pl_dma_variant - PL DMA PCIe variant information + * @version: DMA version + */ +struct xilinx_pl_dma_variant { + enum xilinx_pl_dma_version version; +}; + struct xilinx_msi { struct irq_domain *msi_domain; unsigned long *bitmap; @@ -88,6 +102,7 @@ struct xilinx_msi { * struct pl_dma_pcie - PCIe port information * @dev: Device pointer * @reg_base: IO Mapped Register Base + * @cfg_base: IO Mapped Configuration Base * @irq: Interrupt number * @cfg: Holds mappings of config space window * @phys_reg_base: Physical address of reg base @@ -97,10 +112,12 @@ struct xilinx_msi { * @msi: MSI information * @intx_irq: INTx error interrupt number * @lock: Lock protecting shared register access + * @variant: PL DMA PCIe version check pointer */ struct pl_dma_pcie { struct device *dev; void __iomem *reg_base; + void __iomem *cfg_base; int irq; struct pci_config_window *cfg; phys_addr_t phys_reg_base; @@ -110,16 +127,23 @@ struct pl_dma_pcie { struct xilinx_msi msi; int intx_irq; raw_spinlock_t lock; + const struct xilinx_pl_dma_variant *variant; }; static inline u32 pcie_read(struct pl_dma_pcie *port, u32 reg) { + if (port->variant->version == QDMA) + return readl(port->reg_base + reg + QDMA_BRIDGE_BASE_OFF); + return readl(port->reg_base + reg); } static inline void pcie_write(struct pl_dma_pcie *port, u32 val, u32 reg) { - writel(val, port->reg_base + reg); + if (port->variant->version == QDMA) + writel(val, port->reg_base + reg + QDMA_BRIDGE_BASE_OFF); + else + writel(val, port->reg_base + reg); } static inline bool xilinx_pl_dma_pcie_link_up(struct pl_dma_pcie *port) @@ -173,6 +197,9 @@ static void __iomem *xilinx_pl_dma_pcie_map_bus(struct pci_bus *bus, if (!xilinx_pl_dma_pcie_valid_device(bus, devfn)) return NULL; + if (port->variant->version == QDMA) + return port->cfg_base + PCIE_ECAM_OFFSET(bus->number, devfn, where); + return port->reg_base + PCIE_ECAM_OFFSET(bus->number, devfn, where); } @@ -731,6 +758,15 @@ static int xilinx_pl_dma_pcie_parse_dt(struct pl_dma_pcie *port, port->reg_base = port->cfg->win; + if (port->variant->version == QDMA) { + port->cfg_base = port->cfg->win; + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "breg"); + port->reg_base = devm_ioremap_resource(dev, res); + if (IS_ERR(port->reg_base)) + return PTR_ERR(port->reg_base); + port->phys_reg_base = res->start; + } + err = xilinx_request_msi_irq(port); if (err) { pci_ecam_free(port->cfg); @@ -760,6 +796,8 @@ static int xilinx_pl_dma_pcie_probe(struct platform_device *pdev) if (!bus) return -ENODEV; + port->variant = of_device_get_match_data(dev); + err = xilinx_pl_dma_pcie_parse_dt(port, bus->res); if (err) { dev_err(dev, "Parsing DT failed\n"); @@ -791,9 +829,22 @@ err_irq_domain: return err; } +static const struct xilinx_pl_dma_variant xdma_host = { + .version = XDMA, +}; + +static const struct xilinx_pl_dma_variant qdma_host = { + .version = QDMA, +}; + static const struct of_device_id xilinx_pl_dma_pcie_of_match[] = { { .compatible = "xlnx,xdma-host-3.00", + .data = &xdma_host, + }, + { + .compatible = "xlnx,qdma-host-3.00", + .data = &qdma_host, }, {} }; From cb08c3a32be4283d650d9f312f8314dbf7cf87ed Mon Sep 17 00:00:00 2001 From: Siddharth Vadapalli Date: Thu, 29 Aug 2024 16:23:15 +0530 Subject: [PATCH 048/302] dt-bindings: PCI: ti,j721e-pci-host: Add ACSPCIE proxy control property MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the "ti,syscon-acspcie-proxy-ctrl" device-tree property which is used to obtain a reference to the ACSPCIE Proxy Control register along with the details of the PAD IO Buffer output enable bits. The ACSPCIE Proxy Control register is used to drive the reference clock for the PCIe Endpoint device via the PAD IO Buffers of the ACSPCIE module. The ACSPCIE module can be used as an alternative to either an on-board clock generator or an external clock generator for providing the reference clock to the PCIe Endpoint device. Link: https://lore.kernel.org/linux-pci/20240829105316.1483684-2-s-vadapalli@ti.com Signed-off-by: Siddharth Vadapalli Signed-off-by: Krzysztof Wilczyński Reviewed-by: Manivannan Sadhasivam Acked-by: Krzysztof Kozlowski --- .../devicetree/bindings/pci/ti,j721e-pci-host.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Documentation/devicetree/bindings/pci/ti,j721e-pci-host.yaml b/Documentation/devicetree/bindings/pci/ti,j721e-pci-host.yaml index 15a2658ceeef..69b499c96c71 100644 --- a/Documentation/devicetree/bindings/pci/ti,j721e-pci-host.yaml +++ b/Documentation/devicetree/bindings/pci/ti,j721e-pci-host.yaml @@ -38,6 +38,16 @@ properties: - const: reg - const: cfg + ti,syscon-acspcie-proxy-ctrl: + $ref: /schemas/types.yaml#/definitions/phandle-array + items: + - items: + - description: Phandle to the ACSPCIE Proxy Control Register + - description: Bitmask corresponding to the PAD IO Buffer + output enable fields (Active Low). + description: Specifier for enabling the ACSPCIE PAD outputs to drive + the reference clock to the Endpoint device. + ti,syscon-pcie-ctrl: $ref: /schemas/types.yaml#/definitions/phandle-array items: From 82c4be4168e26a5593aaa1002b5678128a638824 Mon Sep 17 00:00:00 2001 From: Siddharth Vadapalli Date: Thu, 29 Aug 2024 16:23:16 +0530 Subject: [PATCH 049/302] PCI: j721e: Enable ACSPCIE Refclk if "ti,syscon-acspcie-proxy-ctrl" exists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ACSPCIE module is capable of driving the reference clock required by the PCIe Endpoint device. It is an alternative to on-board and external reference clock generators. Enabling the output from the ACSPCIE module's PAD IO Buffers requires clearing the "PAD IO disable" bits of the ACSPCIE_PROXY_CTRL register in the CTRL_MMR register space. Add support to enable the ACSPCIE reference clock output using the optional device-tree property "ti,syscon-acspcie-proxy-ctrl". Link: https://lore.kernel.org/linux-pci/20240829105316.1483684-3-s-vadapalli@ti.com Signed-off-by: Siddharth Vadapalli Signed-off-by: Krzysztof Wilczyński Reviewed-by: Manivannan Sadhasivam --- drivers/pci/controller/cadence/pci-j721e.c | 39 +++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/drivers/pci/controller/cadence/pci-j721e.c b/drivers/pci/controller/cadence/pci-j721e.c index 85718246016b..7f7732e2dcaa 100644 --- a/drivers/pci/controller/cadence/pci-j721e.c +++ b/drivers/pci/controller/cadence/pci-j721e.c @@ -44,6 +44,7 @@ enum link_status { #define J721E_MODE_RC BIT(7) #define LANE_COUNT(n) ((n) << 8) +#define ACSPCIE_PAD_DISABLE_MASK GENMASK(1, 0) #define GENERATION_SEL_MASK GENMASK(1, 0) struct j721e_pcie { @@ -220,6 +221,36 @@ static int j721e_pcie_set_lane_count(struct j721e_pcie *pcie, return ret; } +static int j721e_enable_acspcie_refclk(struct j721e_pcie *pcie, + struct regmap *syscon) +{ + struct device *dev = pcie->cdns_pcie->dev; + struct device_node *node = dev->of_node; + u32 mask = ACSPCIE_PAD_DISABLE_MASK; + struct of_phandle_args args; + u32 val; + int ret; + + ret = of_parse_phandle_with_fixed_args(node, + "ti,syscon-acspcie-proxy-ctrl", + 1, 0, &args); + if (ret) { + dev_err(dev, + "ti,syscon-acspcie-proxy-ctrl has invalid arguments\n"); + return ret; + } + + /* Clear PAD IO disable bits to enable refclk output */ + val = ~(args.args[0]); + ret = regmap_update_bits(syscon, 0, mask, val); + if (ret) { + dev_err(dev, "failed to enable ACSPCIE refclk: %d\n", ret); + return ret; + } + + return 0; +} + static int j721e_pcie_ctrl_init(struct j721e_pcie *pcie) { struct device *dev = pcie->cdns_pcie->dev; @@ -259,7 +290,13 @@ static int j721e_pcie_ctrl_init(struct j721e_pcie *pcie) return ret; } - return 0; + /* Enable ACSPCIE refclk output if the optional property exists */ + syscon = syscon_regmap_lookup_by_phandle_optional(node, + "ti,syscon-acspcie-proxy-ctrl"); + if (!syscon) + return 0; + + return j721e_enable_acspcie_refclk(pcie, syscon); } static int cdns_ti_pcie_config_read(struct pci_bus *bus, unsigned int devfn, From 3858e8a5ea719411d6682e2218c41eff27092463 Mon Sep 17 00:00:00 2001 From: Manivannan Sadhasivam Date: Wed, 28 Aug 2024 21:16:11 +0530 Subject: [PATCH 050/302] PCI: qcom-ep: Drop the redundant masking of global IRQ events MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Once the events are disabled in PARF_INT_ALL_MASK register, only the enabled events will generate global IRQ. So there is no need to do the masking again in the IRQ handler, drop it. If there are any spurious IRQs getting generated, they will be reported using the existing dev_err() in the handler. Link: https://lore.kernel.org/linux-pci/20240828-pci-qcom-hotplug-v4-1-263a385fbbcb@linaro.org Signed-off-by: Manivannan Sadhasivam Signed-off-by: Krzysztof Wilczyński Reviewed-by: Konrad Dybcio --- drivers/pci/controller/dwc/pcie-qcom-ep.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/pci/controller/dwc/pcie-qcom-ep.c b/drivers/pci/controller/dwc/pcie-qcom-ep.c index 236229f66c80..972a90eba494 100644 --- a/drivers/pci/controller/dwc/pcie-qcom-ep.c +++ b/drivers/pci/controller/dwc/pcie-qcom-ep.c @@ -647,11 +647,9 @@ static irqreturn_t qcom_pcie_ep_global_irq_thread(int irq, void *data) struct dw_pcie *pci = &pcie_ep->pci; struct device *dev = pci->dev; u32 status = readl_relaxed(pcie_ep->parf + PARF_INT_ALL_STATUS); - u32 mask = readl_relaxed(pcie_ep->parf + PARF_INT_ALL_MASK); u32 dstate, val; writel_relaxed(status, pcie_ep->parf + PARF_INT_ALL_CLEAR); - status &= mask; if (FIELD_GET(PARF_INT_ALL_LINK_DOWN, status)) { dev_dbg(dev, "Received Linkdown event\n"); From 95bebcbd657cf8eb86025a2b20961e43d2e433f5 Mon Sep 17 00:00:00 2001 From: Manivannan Sadhasivam Date: Wed, 28 Aug 2024 21:16:12 +0530 Subject: [PATCH 051/302] PCI: qcom-ep: Reword the error message for receiving unknown global IRQ event MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Current error message just prints the contents of PARF_INT_ALL_STATUS register as if like the IRQ event number. It could mislead the users. Reword it to make it clear that the error message is actually showing the interrupt status register to help debug spurious IRQ events. While at it, let's also switch over to dev_WARN_ONCE() so that any IRQ storm won't flood the kernel log buffer. Link: https://lore.kernel.org/linux-pci/20240828-pci-qcom-hotplug-v4-2-263a385fbbcb@linaro.org Signed-off-by: Manivannan Sadhasivam Signed-off-by: Krzysztof Wilczyński Reviewed-by: Konrad Dybcio --- drivers/pci/controller/dwc/pcie-qcom-ep.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/pci/controller/dwc/pcie-qcom-ep.c b/drivers/pci/controller/dwc/pcie-qcom-ep.c index 972a90eba494..0bb0a056dd8f 100644 --- a/drivers/pci/controller/dwc/pcie-qcom-ep.c +++ b/drivers/pci/controller/dwc/pcie-qcom-ep.c @@ -679,7 +679,8 @@ static irqreturn_t qcom_pcie_ep_global_irq_thread(int irq, void *data) dw_pcie_ep_linkup(&pci->ep); pcie_ep->link_status = QCOM_PCIE_EP_LINK_UP; } else { - dev_err(dev, "Received unknown event: %d\n", status); + dev_WARN_ONCE(dev, 1, "Received unknown event. INT_STATUS: 0x%08x\n", + status); } return IRQ_HANDLED; From 99244b999deca227a4c035f93a06d94fe7921a80 Mon Sep 17 00:00:00 2001 From: Manivannan Sadhasivam Date: Wed, 28 Aug 2024 21:16:13 +0530 Subject: [PATCH 052/302] dt-bindings: PCI: pci-ep: Update Maintainers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kishon's TI email ID is not active anymore, so use his kernel.org ID. Also, since I've been maintaining the PCI endpoint framework, I'm willing to maintain the DT binding as well. So add myself as the co-maintainer. Link: https://lore.kernel.org/linux-pci/20240828-pci-qcom-hotplug-v4-3-263a385fbbcb@linaro.org Signed-off-by: Manivannan Sadhasivam [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Frank Li Acked-by: Rob Herring (Arm) --- Documentation/devicetree/bindings/pci/pci-ep.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/pci/pci-ep.yaml b/Documentation/devicetree/bindings/pci/pci-ep.yaml index d1eef4825207..0b5456ee21eb 100644 --- a/Documentation/devicetree/bindings/pci/pci-ep.yaml +++ b/Documentation/devicetree/bindings/pci/pci-ep.yaml @@ -10,7 +10,8 @@ description: | Common properties for PCI Endpoint Controller Nodes. maintainers: - - Kishon Vijay Abraham I + - Kishon Vijay Abraham I + - Manivannan Sadhasivam properties: $nodename: From ada94d00620a1a5b52ddf41bbbe3b767a10bc2ea Mon Sep 17 00:00:00 2001 From: Manivannan Sadhasivam Date: Wed, 28 Aug 2024 21:16:14 +0530 Subject: [PATCH 053/302] dt-bindings: PCI: pci-ep: Document 'linux,pci-domain' property MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 'linux,pci-domain' property provides the PCI domain number for the PCI endpoint controllers in a SoC. If this property is not present, then an unstable (across boots) unique number will be assigned. Devicetrees can specify the domain number based on the actual hardware instance of the PCI endpoint controllers in the SoC. Link: https://lore.kernel.org/linux-pci/20240828-pci-qcom-hotplug-v4-4-263a385fbbcb@linaro.org Signed-off-by: Manivannan Sadhasivam Signed-off-by: Krzysztof Wilczyński Reviewed-by: Rob Herring (Arm) --- Documentation/devicetree/bindings/pci/pci-ep.yaml | 11 +++++++++++ .../devicetree/bindings/pci/qcom,pcie-ep.yaml | 1 + 2 files changed, 12 insertions(+) diff --git a/Documentation/devicetree/bindings/pci/pci-ep.yaml b/Documentation/devicetree/bindings/pci/pci-ep.yaml index 0b5456ee21eb..f75000e3093d 100644 --- a/Documentation/devicetree/bindings/pci/pci-ep.yaml +++ b/Documentation/devicetree/bindings/pci/pci-ep.yaml @@ -42,6 +42,17 @@ properties: default: 1 maximum: 16 + linux,pci-domain: + description: + If present this property assigns a fixed PCI domain number to a PCI + Endpoint Controller, otherwise an unstable (across boots) unique number + will be assigned. It is required to either not set this property at all + or set it for all PCI endpoint controllers in the system, otherwise + potentially conflicting domain numbers may be assigned to endpoint + controllers. The domain number for each endpoint controller in the system + must be unique. + $ref: /schemas/types.yaml#/definitions/uint32 + required: - compatible diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie-ep.yaml b/Documentation/devicetree/bindings/pci/qcom,pcie-ep.yaml index 46802f7d9482..1226ee5d08d1 100644 --- a/Documentation/devicetree/bindings/pci/qcom,pcie-ep.yaml +++ b/Documentation/devicetree/bindings/pci/qcom,pcie-ep.yaml @@ -280,4 +280,5 @@ examples: phy-names = "pciephy"; max-link-speed = <3>; num-lanes = <2>; + linux,pci-domain = <0>; }; From 0328947c50324cf4b2d8b181bf948edb8101f59f Mon Sep 17 00:00:00 2001 From: Manivannan Sadhasivam Date: Wed, 28 Aug 2024 21:16:15 +0530 Subject: [PATCH 054/302] PCI: endpoint: Assign PCI domain number for endpoint controllers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Right now, PCI endpoint subsystem doesn't assign PCI domain number for the PCI endpoint controllers. But this domain number could be useful to the EPC drivers to uniquely identify each controller based on the hardware instance when there are multiple ones present in an SoC (even multiple RC/EP). So let's make use of the existing pci_bus_find_domain_nr() API to allocate domain numbers based on either devicetree (linux,pci-domain) property or dynamic domain number allocation scheme. It should be noted that the domain number allocated by this API will be based on both RC and EP controllers in a SoC. If the 'linux,pci-domain' DT property is present, then the domain number represents the actual hardware instance of the PCI endpoint controller. If not, then the domain number will be allocated based on the PCI EP/RC controller probe order. If the architecture doesn't support CONFIG_PCI_DOMAINS_GENERIC (rare), then currently a warning is thrown to indicate that the architecture specific implementation is needed. Link: https://lore.kernel.org/linux-pci/20240828-pci-qcom-hotplug-v4-5-263a385fbbcb@linaro.org Signed-off-by: Manivannan Sadhasivam Signed-off-by: Krzysztof Wilczyński Reviewed-by: Frank Li --- drivers/pci/endpoint/pci-epc-core.c | 14 ++++++++++++++ include/linux/pci-epc.h | 2 ++ 2 files changed, 16 insertions(+) diff --git a/drivers/pci/endpoint/pci-epc-core.c b/drivers/pci/endpoint/pci-epc-core.c index 84309dfe0c68..085a2de8b923 100644 --- a/drivers/pci/endpoint/pci-epc-core.c +++ b/drivers/pci/endpoint/pci-epc-core.c @@ -838,6 +838,10 @@ void pci_epc_destroy(struct pci_epc *epc) { pci_ep_cfs_remove_epc_group(epc->group); device_unregister(&epc->dev); + +#ifdef CONFIG_PCI_DOMAINS_GENERIC + pci_bus_release_domain_nr(NULL, &epc->dev); +#endif } EXPORT_SYMBOL_GPL(pci_epc_destroy); @@ -900,6 +904,16 @@ __pci_epc_create(struct device *dev, const struct pci_epc_ops *ops, epc->dev.release = pci_epc_release; epc->ops = ops; +#ifdef CONFIG_PCI_DOMAINS_GENERIC + epc->domain_nr = pci_bus_find_domain_nr(NULL, dev); +#else + /* + * TODO: If the architecture doesn't support generic PCI + * domains, then a custom implementation has to be used. + */ + WARN_ONCE(1, "This architecture doesn't support generic PCI domains\n"); +#endif + ret = dev_set_name(&epc->dev, "%s", dev_name(dev)); if (ret) goto put_dev; diff --git a/include/linux/pci-epc.h b/include/linux/pci-epc.h index 85bdf2adb760..8e3dcac55dcd 100644 --- a/include/linux/pci-epc.h +++ b/include/linux/pci-epc.h @@ -128,6 +128,7 @@ struct pci_epc_mem { * @group: configfs group representing the PCI EPC device * @lock: mutex to protect pci_epc ops * @function_num_map: bitmap to manage physical function number + * @domain_nr: PCI domain number of the endpoint controller * @init_complete: flag to indicate whether the EPC initialization is complete * or not */ @@ -145,6 +146,7 @@ struct pci_epc { /* mutex to protect against concurrent access of EP controller */ struct mutex lock; unsigned long function_num_map; + int domain_nr; bool init_complete; }; From bba1251edf8501f85304b7d65ae2aac309f2d0a1 Mon Sep 17 00:00:00 2001 From: Manivannan Sadhasivam Date: Wed, 28 Aug 2024 21:16:16 +0530 Subject: [PATCH 055/302] PCI: qcom-ep: Modify 'global_irq' and 'perst_irq' IRQ device names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently, the IRQ device name for both of these IRQs doesn't have Qcom specific prefix and PCIe domain number. This causes 2 issues: 1. Pollutes the global IRQ namespace since 'global' is a common name. 2. When more than one EP controller instance is present in the SoC, naming conflict will occur. Hence, add 'qcom_pcie_ep_' prefix and PCIe domain number suffix to the IRQ names to uniquely identify the IRQs and also to fix the above mentioned issues. Link: https://lore.kernel.org/linux-pci/20240828-pci-qcom-hotplug-v4-6-263a385fbbcb@linaro.org Signed-off-by: Manivannan Sadhasivam Signed-off-by: Krzysztof Wilczyński Reviewed-by: Konrad Dybcio --- drivers/pci/controller/dwc/pcie-qcom-ep.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/drivers/pci/controller/dwc/pcie-qcom-ep.c b/drivers/pci/controller/dwc/pcie-qcom-ep.c index 0bb0a056dd8f..d0a27fa6fdc8 100644 --- a/drivers/pci/controller/dwc/pcie-qcom-ep.c +++ b/drivers/pci/controller/dwc/pcie-qcom-ep.c @@ -711,8 +711,15 @@ static irqreturn_t qcom_pcie_ep_perst_irq_thread(int irq, void *data) static int qcom_pcie_ep_enable_irq_resources(struct platform_device *pdev, struct qcom_pcie_ep *pcie_ep) { + struct device *dev = pcie_ep->pci.dev; + char *name; int ret; + name = devm_kasprintf(dev, GFP_KERNEL, "qcom_pcie_ep_global_irq%d", + pcie_ep->pci.ep.epc->domain_nr); + if (!name) + return -ENOMEM; + pcie_ep->global_irq = platform_get_irq_byname(pdev, "global"); if (pcie_ep->global_irq < 0) return pcie_ep->global_irq; @@ -720,18 +727,23 @@ static int qcom_pcie_ep_enable_irq_resources(struct platform_device *pdev, ret = devm_request_threaded_irq(&pdev->dev, pcie_ep->global_irq, NULL, qcom_pcie_ep_global_irq_thread, IRQF_ONESHOT, - "global_irq", pcie_ep); + name, pcie_ep); if (ret) { dev_err(&pdev->dev, "Failed to request Global IRQ\n"); return ret; } + name = devm_kasprintf(dev, GFP_KERNEL, "qcom_pcie_ep_perst_irq%d", + pcie_ep->pci.ep.epc->domain_nr); + if (!name) + return -ENOMEM; + pcie_ep->perst_irq = gpiod_to_irq(pcie_ep->reset); irq_set_status_flags(pcie_ep->perst_irq, IRQ_NOAUTOEN); ret = devm_request_threaded_irq(&pdev->dev, pcie_ep->perst_irq, NULL, qcom_pcie_ep_perst_irq_thread, IRQF_TRIGGER_HIGH | IRQF_ONESHOT, - "perst_irq", pcie_ep); + name, pcie_ep); if (ret) { dev_err(&pdev->dev, "Failed to request PERST IRQ\n"); disable_irq(pcie_ep->global_irq); From 6efd853303a5eea90a21a1ec496005317b6cc2e3 Mon Sep 17 00:00:00 2001 From: Manivannan Sadhasivam Date: Wed, 28 Aug 2024 21:16:20 +0530 Subject: [PATCH 056/302] dt-bindings: PCI: qcom,pcie-sm8450: Add 'global' interrupt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Qcom PCIe RC controllers are capable of generating 'global' SPI interrupt to the host CPU. This interrupt can be used by the device driver to identify events such as PCIe link specific events, safety events, etc... Hence, document it in the binding along with the existing MSI interrupts. Though adding a new interrupt will break the ABI, it is required to accurately describe the hardware. Link: https://lore.kernel.org/linux-pci/20240828-pci-qcom-hotplug-v4-10-263a385fbbcb@linaro.org Signed-off-by: Manivannan Sadhasivam Signed-off-by: Krzysztof Wilczyński Reviewed-by: Rob Herring (Arm) --- .../devicetree/bindings/pci/qcom,pcie-common.yaml | 4 ++-- .../devicetree/bindings/pci/qcom,pcie-sm8450.yaml | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie-common.yaml b/Documentation/devicetree/bindings/pci/qcom,pcie-common.yaml index 0a39bbfcb28b..704c0f58eea5 100644 --- a/Documentation/devicetree/bindings/pci/qcom,pcie-common.yaml +++ b/Documentation/devicetree/bindings/pci/qcom,pcie-common.yaml @@ -21,11 +21,11 @@ properties: interrupts: minItems: 1 - maxItems: 8 + maxItems: 9 interrupt-names: minItems: 1 - maxItems: 8 + maxItems: 9 iommu-map: minItems: 1 diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie-sm8450.yaml b/Documentation/devicetree/bindings/pci/qcom,pcie-sm8450.yaml index d8c0afaa4b19..46bd59eefadb 100644 --- a/Documentation/devicetree/bindings/pci/qcom,pcie-sm8450.yaml +++ b/Documentation/devicetree/bindings/pci/qcom,pcie-sm8450.yaml @@ -55,8 +55,8 @@ properties: - const: aggre1 # Aggre NoC PCIe1 AXI clock interrupts: - minItems: 8 - maxItems: 8 + minItems: 9 + maxItems: 9 interrupt-names: items: @@ -68,6 +68,7 @@ properties: - const: msi5 - const: msi6 - const: msi7 + - const: global operating-points-v2: true opp-table: @@ -149,9 +150,10 @@ examples: , , , - ; + , + ; interrupt-names = "msi0", "msi1", "msi2", "msi3", - "msi4", "msi5", "msi6", "msi7"; + "msi4", "msi5", "msi6", "msi7", "global"; #interrupt-cells = <1>; interrupt-map-mask = <0 0 0 0x7>; interrupt-map = <0 0 0 1 &intc 0 0 0 149 IRQ_TYPE_LEVEL_HIGH>, /* int_a */ From 4581403f67929d02c197cb187c4e1e811c9e762a Mon Sep 17 00:00:00 2001 From: Manivannan Sadhasivam Date: Wed, 28 Aug 2024 21:16:21 +0530 Subject: [PATCH 057/302] PCI: qcom: Enumerate endpoints based on Link up event in 'global_irq' interrupt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Historically, Qcom PCIe RC controllers lacked standard hotplug support. So when an endpoint is attached to the SoC, users have to rescan the bus manually to enumerate the device. But this can be avoided by using the Link up event exposed by the Qcom specific 'global_irq' interrupt. Qcom PCIe RC controllers are capable of generating the 'global' SPI interrupt to the host CPUs. The device driver can use this interrupt to identify events such as PCIe link specific events, safety events etc... One such event is the PCIe Link up event generated when an endpoint is detected on the bus and the Link is 'up'. This event can be used to enumerate the PCIe endpoint devices without user intervention. So add support for capturing the PCIe Link up event using the 'global' interrupt in the driver. Once the Link up event is received, the bus underneath the host bridge is scanned to enumerate PCIe endpoint devices. All of the Qcom SoCs have only one Root Port per controller instance. So only a single 'Link up' event is generated for the PCIe controller. Link: https://lore.kernel.org/linux-pci/20240828-pci-qcom-hotplug-v4-11-263a385fbbcb@linaro.org Signed-off-by: Manivannan Sadhasivam Signed-off-by: Krzysztof Wilczyński Reviewed-by: Konrad Dybcio --- drivers/pci/controller/dwc/pcie-qcom.c | 55 +++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c index 0180edf3310e..a1d678fe7fa5 100644 --- a/drivers/pci/controller/dwc/pcie-qcom.c +++ b/drivers/pci/controller/dwc/pcie-qcom.c @@ -50,6 +50,9 @@ #define PARF_AXI_MSTR_WR_ADDR_HALT_V2 0x1a8 #define PARF_Q2A_FLUSH 0x1ac #define PARF_LTSSM 0x1b0 +#define PARF_INT_ALL_STATUS 0x224 +#define PARF_INT_ALL_CLEAR 0x228 +#define PARF_INT_ALL_MASK 0x22c #define PARF_SID_OFFSET 0x234 #define PARF_BDF_TRANSLATE_CFG 0x24c #define PARF_SLV_ADDR_SPACE_SIZE 0x358 @@ -121,6 +124,9 @@ /* PARF_LTSSM register fields */ #define LTSSM_EN BIT(8) +/* PARF_INT_ALL_{STATUS/CLEAR/MASK} register fields */ +#define PARF_INT_ALL_LINK_UP BIT(13) + /* PARF_NO_SNOOP_OVERIDE register fields */ #define WR_NO_SNOOP_OVERIDE_EN BIT(1) #define RD_NO_SNOOP_OVERIDE_EN BIT(3) @@ -1488,6 +1494,29 @@ static void qcom_pcie_init_debugfs(struct qcom_pcie *pcie) qcom_pcie_link_transition_count); } +static irqreturn_t qcom_pcie_global_irq_thread(int irq, void *data) +{ + struct qcom_pcie *pcie = data; + struct dw_pcie_rp *pp = &pcie->pci->pp; + struct device *dev = pcie->pci->dev; + u32 status = readl_relaxed(pcie->parf + PARF_INT_ALL_STATUS); + + writel_relaxed(status, pcie->parf + PARF_INT_ALL_CLEAR); + + if (FIELD_GET(PARF_INT_ALL_LINK_UP, status)) { + dev_dbg(dev, "Received Link up event. Starting enumeration!\n"); + /* Rescan the bus to enumerate endpoint devices */ + pci_lock_rescan_remove(); + pci_rescan_bus(pp->bridge->bus); + pci_unlock_rescan_remove(); + } else { + dev_WARN_ONCE(dev, 1, "Received unknown event. INT_STATUS: 0x%08x\n", + status); + } + + return IRQ_HANDLED; +} + static int qcom_pcie_probe(struct platform_device *pdev) { const struct qcom_pcie_cfg *pcie_cfg; @@ -1498,7 +1527,8 @@ static int qcom_pcie_probe(struct platform_device *pdev) struct dw_pcie_rp *pp; struct resource *res; struct dw_pcie *pci; - int ret; + int ret, irq; + char *name; pcie_cfg = of_device_get_match_data(dev); if (!pcie_cfg || !pcie_cfg->ops) { @@ -1617,6 +1647,27 @@ static int qcom_pcie_probe(struct platform_device *pdev) goto err_phy_exit; } + name = devm_kasprintf(dev, GFP_KERNEL, "qcom_pcie_global_irq%d", + pci_domain_nr(pp->bridge->bus)); + if (!name) { + ret = -ENOMEM; + goto err_host_deinit; + } + + irq = platform_get_irq_byname_optional(pdev, "global"); + if (irq > 0) { + ret = devm_request_threaded_irq(&pdev->dev, irq, NULL, + qcom_pcie_global_irq_thread, + IRQF_ONESHOT, name, pcie); + if (ret) { + dev_err_probe(&pdev->dev, ret, + "Failed to request Global IRQ\n"); + goto err_host_deinit; + } + + writel_relaxed(PARF_INT_ALL_LINK_UP, pcie->parf + PARF_INT_ALL_MASK); + } + qcom_pcie_icc_opp_update(pcie); if (pcie->mhi) @@ -1624,6 +1675,8 @@ static int qcom_pcie_probe(struct platform_device *pdev) return 0; +err_host_deinit: + dw_pcie_host_deinit(pp); err_phy_exit: phy_exit(pcie->phy); err_pm_runtime_put: From 10ba0854c5e6165b58e17bda5fb671e729fecf9e Mon Sep 17 00:00:00 2001 From: Prudhvi Yarlagadda Date: Wed, 14 Aug 2024 15:03:38 -0700 Subject: [PATCH 058/302] PCI: qcom: Disable mirroring of DBI and iATU register space in BAR region MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PARF hardware block which is a wrapper on top of DWC PCIe controller mirrors the DBI and ATU register space. It uses PARF_SLV_ADDR_SPACE_SIZE register to get the size of the memory block to be mirrored and uses PARF_DBI_BASE_ADDR, PARF_ATU_BASE_ADDR registers to determine the base address of DBI and ATU space inside the memory block that is being mirrored. When a memory region which is located above the SLV_ADDR_SPACE_SIZE boundary is used for BAR region then there could be an overlap of DBI and ATU address space that is getting mirrored and the BAR region. This results in DBI and ATU address space contents getting updated when a PCIe function driver tries updating the BAR/MMIO memory region. Reference memory map of the PCIe memory region with DBI and ATU address space overlapping BAR region is as below. |---------------| | | | | ------- --------|---------------| | | |---------------| | | | DBI | | | |---------------|---->DBI_BASE_ADDR | | | | | | | | | PCIe | |---->2*SLV_ADDR_SPACE_SIZE | BAR/MMIO|---------------| | Region | ATU | | | |---------------|---->ATU_BASE_ADDR | | | | PCIe | |---------------| Memory | | DBI | Region | |---------------|---->DBI_BASE_ADDR | | | | | --------| | | | |---->SLV_ADDR_SPACE_SIZE | |---------------| | | ATU | | |---------------|---->ATU_BASE_ADDR | | | | |---------------| | | DBI | | |---------------|---->DBI_BASE_ADDR | | | | | | ----------------|---------------| | | | | | | |---------------| Currently memory region beyond the SLV_ADDR_SPACE_SIZE boundary is not used for BAR region which is why the above mentioned issue is not encountered. This issue is discovered as part of internal testing when we tried moving the BAR region beyond the SLV_ADDR_SPACE_SIZE boundary. Hence we are trying to fix this. As PARF hardware block mirrors DBI and ATU register space after every PARF_SLV_ADDR_SPACE_SIZE (default 0x1000000) boundary multiple, program maximum possible size to this register by writing 0x80000000 to it(it considers only powers of 2 as values) to avoid mirroring DBI and ATU to BAR/MMIO region. Write the physical base address of DBI and ATU register blocks to PARF_DBI_BASE_ADDR (default 0x0) and PARF_ATU_BASE_ADDR (default 0x1000) respectively to make sure DBI and ATU blocks are at expected memory locations. The register offsets PARF_DBI_BASE_ADDR_V2, PARF_SLV_ADDR_SPACE_SIZE_V2 and PARF_ATU_BASE_ADDR are applicable for platforms that use Qcom IP rev 1.9.0, 2.7.0 and 2.9.0. PARF_DBI_BASE_ADDR_V2 and PARF_SLV_ADDR_SPACE_SIZE_V2 are applicable for Qcom IP rev 2.3.3. PARF_DBI_BASE_ADDR and PARF_SLV_ADDR_SPACE_SIZE are applicable for Qcom IP rev 1.0.0, 2.3.2 and 2.4.0. Update init()/post_init() functions of the respective Qcom IP versions to program applicable PARF_DBI_BASE_ADDR, PARF_SLV_ADDR_SPACE_SIZE and PARF_ATU_BASE_ADDR register offsets. Update the SLV_ADDR_SPACE_SZ macro to 0x80000000 to set highest bit in PARF_SLV_ADDR_SPACE_SIZE register. Cache DBI and iATU physical addresses in 'struct dw_pcie' so that pcie_qcom.c driver can program these addresses in the PARF_DBI_BASE_ADDR and PARF_ATU_BASE_ADDR registers. Suggested-by: Manivannan Sadhasivam Link: https://lore.kernel.org/linux-pci/20240814220338.1969668-1-quic_pyarlaga@quicinc.com Signed-off-by: Prudhvi Yarlagadda Signed-off-by: Krzysztof Wilczyński Reviewed-by: Manivannan Sadhasivam Reviewed-by: Mayank Rana --- drivers/pci/controller/dwc/pcie-designware.c | 2 + drivers/pci/controller/dwc/pcie-designware.h | 2 + drivers/pci/controller/dwc/pcie-qcom.c | 72 ++++++++++++++++---- 3 files changed, 61 insertions(+), 15 deletions(-) diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c index 1b5aba1f0c92..bc3a5d6b0177 100644 --- a/drivers/pci/controller/dwc/pcie-designware.c +++ b/drivers/pci/controller/dwc/pcie-designware.c @@ -112,6 +112,7 @@ int dw_pcie_get_resources(struct dw_pcie *pci) pci->dbi_base = devm_pci_remap_cfg_resource(pci->dev, res); if (IS_ERR(pci->dbi_base)) return PTR_ERR(pci->dbi_base); + pci->dbi_phys_addr = res->start; } /* DBI2 is mainly useful for the endpoint controller */ @@ -134,6 +135,7 @@ int dw_pcie_get_resources(struct dw_pcie *pci) pci->atu_base = devm_ioremap_resource(pci->dev, res); if (IS_ERR(pci->atu_base)) return PTR_ERR(pci->atu_base); + pci->atu_phys_addr = res->start; } else { pci->atu_base = pci->dbi_base + DEFAULT_DBI_ATU_OFFSET; } diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h index 53c4c8f399c8..e518f81ea80c 100644 --- a/drivers/pci/controller/dwc/pcie-designware.h +++ b/drivers/pci/controller/dwc/pcie-designware.h @@ -407,8 +407,10 @@ struct dw_pcie_ops { struct dw_pcie { struct device *dev; void __iomem *dbi_base; + resource_size_t dbi_phys_addr; void __iomem *dbi_base2; void __iomem *atu_base; + resource_size_t atu_phys_addr; size_t atu_size; u32 num_ib_windows; u32 num_ob_windows; diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c index a1d678fe7fa5..1923266acea8 100644 --- a/drivers/pci/controller/dwc/pcie-qcom.c +++ b/drivers/pci/controller/dwc/pcie-qcom.c @@ -45,6 +45,7 @@ #define PARF_PHY_REFCLK 0x4c #define PARF_CONFIG_BITS 0x50 #define PARF_DBI_BASE_ADDR 0x168 +#define PARF_SLV_ADDR_SPACE_SIZE 0x16c #define PARF_MHI_CLOCK_RESET_CTRL 0x174 #define PARF_AXI_MSTR_WR_ADDR_HALT 0x178 #define PARF_AXI_MSTR_WR_ADDR_HALT_V2 0x1a8 @@ -55,8 +56,13 @@ #define PARF_INT_ALL_MASK 0x22c #define PARF_SID_OFFSET 0x234 #define PARF_BDF_TRANSLATE_CFG 0x24c -#define PARF_SLV_ADDR_SPACE_SIZE 0x358 +#define PARF_DBI_BASE_ADDR_V2 0x350 +#define PARF_DBI_BASE_ADDR_V2_HI 0x354 +#define PARF_SLV_ADDR_SPACE_SIZE_V2 0x358 +#define PARF_SLV_ADDR_SPACE_SIZE_V2_HI 0x35c #define PARF_NO_SNOOP_OVERIDE 0x3d4 +#define PARF_ATU_BASE_ADDR 0x634 +#define PARF_ATU_BASE_ADDR_HI 0x638 #define PARF_DEVICE_TYPE 0x1000 #define PARF_BDF_TO_SID_TABLE_N 0x2000 #define PARF_BDF_TO_SID_CFG 0x2c00 @@ -111,7 +117,7 @@ #define PHY_RX0_EQ(x) FIELD_PREP(GENMASK(26, 24), x) /* PARF_SLV_ADDR_SPACE_SIZE register value */ -#define SLV_ADDR_SPACE_SZ 0x10000000 +#define SLV_ADDR_SPACE_SZ 0x80000000 /* PARF_MHI_CLOCK_RESET_CTRL register fields */ #define AHB_CLK_EN BIT(0) @@ -330,6 +336,50 @@ static void qcom_pcie_clear_hpc(struct dw_pcie *pci) dw_pcie_dbi_ro_wr_dis(pci); } +static void qcom_pcie_configure_dbi_base(struct qcom_pcie *pcie) +{ + struct dw_pcie *pci = pcie->pci; + + if (pci->dbi_phys_addr) { + /* + * PARF_DBI_BASE_ADDR register is in CPU domain and require to + * be programmed with CPU physical address. + */ + writel(lower_32_bits(pci->dbi_phys_addr), pcie->parf + + PARF_DBI_BASE_ADDR); + writel(SLV_ADDR_SPACE_SZ, pcie->parf + + PARF_SLV_ADDR_SPACE_SIZE); + } +} + +static void qcom_pcie_configure_dbi_atu_base(struct qcom_pcie *pcie) +{ + struct dw_pcie *pci = pcie->pci; + + if (pci->dbi_phys_addr) { + /* + * PARF_DBI_BASE_ADDR_V2 and PARF_ATU_BASE_ADDR registers are + * in CPU domain and require to be programmed with CPU + * physical addresses. + */ + writel(lower_32_bits(pci->dbi_phys_addr), pcie->parf + + PARF_DBI_BASE_ADDR_V2); + writel(upper_32_bits(pci->dbi_phys_addr), pcie->parf + + PARF_DBI_BASE_ADDR_V2_HI); + + if (pci->atu_phys_addr) { + writel(lower_32_bits(pci->atu_phys_addr), pcie->parf + + PARF_ATU_BASE_ADDR); + writel(upper_32_bits(pci->atu_phys_addr), pcie->parf + + PARF_ATU_BASE_ADDR_HI); + } + + writel(0x0, pcie->parf + PARF_SLV_ADDR_SPACE_SIZE_V2); + writel(SLV_ADDR_SPACE_SZ, pcie->parf + + PARF_SLV_ADDR_SPACE_SIZE_V2_HI); + } +} + static void qcom_pcie_2_1_0_ltssm_enable(struct qcom_pcie *pcie) { u32 val; @@ -546,8 +596,7 @@ err_assert_reset: static int qcom_pcie_post_init_1_0_0(struct qcom_pcie *pcie) { - /* change DBI base address */ - writel(0, pcie->parf + PARF_DBI_BASE_ADDR); + qcom_pcie_configure_dbi_base(pcie); if (IS_ENABLED(CONFIG_PCI_MSI)) { u32 val = readl(pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT); @@ -634,8 +683,7 @@ static int qcom_pcie_post_init_2_3_2(struct qcom_pcie *pcie) val &= ~PHY_TEST_PWR_DOWN; writel(val, pcie->parf + PARF_PHY_CTRL); - /* change DBI base address */ - writel(0, pcie->parf + PARF_DBI_BASE_ADDR); + qcom_pcie_configure_dbi_base(pcie); /* MAC PHY_POWERDOWN MUX DISABLE */ val = readl(pcie->parf + PARF_SYS_CTRL); @@ -817,13 +865,11 @@ static int qcom_pcie_post_init_2_3_3(struct qcom_pcie *pcie) u16 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); u32 val; - writel(SLV_ADDR_SPACE_SZ, pcie->parf + PARF_SLV_ADDR_SPACE_SIZE); - val = readl(pcie->parf + PARF_PHY_CTRL); val &= ~PHY_TEST_PWR_DOWN; writel(val, pcie->parf + PARF_PHY_CTRL); - writel(0, pcie->parf + PARF_DBI_BASE_ADDR); + qcom_pcie_configure_dbi_atu_base(pcie); writel(MST_WAKEUP_EN | SLV_WAKEUP_EN | MSTR_ACLK_CGC_DIS | SLV_ACLK_CGC_DIS | CORE_CLK_CGC_DIS | @@ -919,8 +965,7 @@ static int qcom_pcie_init_2_7_0(struct qcom_pcie *pcie) val &= ~PHY_TEST_PWR_DOWN; writel(val, pcie->parf + PARF_PHY_CTRL); - /* change DBI base address */ - writel(0, pcie->parf + PARF_DBI_BASE_ADDR); + qcom_pcie_configure_dbi_atu_base(pcie); /* MAC PHY_POWERDOWN MUX DISABLE */ val = readl(pcie->parf + PARF_SYS_CTRL); @@ -1129,14 +1174,11 @@ static int qcom_pcie_post_init_2_9_0(struct qcom_pcie *pcie) u32 val; int i; - writel(SLV_ADDR_SPACE_SZ, - pcie->parf + PARF_SLV_ADDR_SPACE_SIZE); - val = readl(pcie->parf + PARF_PHY_CTRL); val &= ~PHY_TEST_PWR_DOWN; writel(val, pcie->parf + PARF_PHY_CTRL); - writel(0, pcie->parf + PARF_DBI_BASE_ADDR); + qcom_pcie_configure_dbi_atu_base(pcie); writel(DEVICE_TYPE_RC, pcie->parf + PARF_DEVICE_TYPE); writel(BYPASS | MSTR_AXI_CLK_EN | AHB_CLK_EN, From 8e6e2ffa6569a205f1805cbaeca143b556581da6 Mon Sep 17 00:00:00 2001 From: Youzhong Yang Date: Wed, 10 Jul 2024 10:40:35 -0400 Subject: [PATCH 059/302] nfsd: add list_head nf_gc to struct nfsd_file nfsd_file_put() in one thread can race with another thread doing garbage collection (running nfsd_file_gc() -> list_lru_walk() -> nfsd_file_lru_cb()): * In nfsd_file_put(), nf->nf_ref is 1, so it tries to do nfsd_file_lru_add(). * nfsd_file_lru_add() returns true (with NFSD_FILE_REFERENCED bit set) * garbage collector kicks in, nfsd_file_lru_cb() clears REFERENCED bit and returns LRU_ROTATE. * garbage collector kicks in again, nfsd_file_lru_cb() now decrements nf->nf_ref to 0, runs nfsd_file_unhash(), removes it from the LRU and adds to the dispose list [list_lru_isolate_move(lru, &nf->nf_lru, head)] * nfsd_file_put() detects NFSD_FILE_HASHED bit is cleared, so it tries to remove the 'nf' from the LRU [if (!nfsd_file_lru_remove(nf))]. The 'nf' has been added to the 'dispose' list by nfsd_file_lru_cb(), so nfsd_file_lru_remove(nf) simply treats it as part of the LRU and removes it, which leads to its removal from the 'dispose' list. * At this moment, 'nf' is unhashed with its nf_ref being 0, and not on the LRU. nfsd_file_put() continues its execution [if (refcount_dec_and_test(&nf->nf_ref))], as nf->nf_ref is already 0, nf->nf_ref is set to REFCOUNT_SATURATED, and the 'nf' gets no chance of being freed. nfsd_file_put() can also race with nfsd_file_cond_queue(): * In nfsd_file_put(), nf->nf_ref is 1, so it tries to do nfsd_file_lru_add(). * nfsd_file_lru_add() sets REFERENCED bit and returns true. * Some userland application runs 'exportfs -f' or something like that, which triggers __nfsd_file_cache_purge() -> nfsd_file_cond_queue(). * In nfsd_file_cond_queue(), it runs [if (!nfsd_file_unhash(nf))], unhash is done successfully. * nfsd_file_cond_queue() runs [if (!nfsd_file_get(nf))], now nf->nf_ref goes to 2. * nfsd_file_cond_queue() runs [if (nfsd_file_lru_remove(nf))], it succeeds. * nfsd_file_cond_queue() runs [if (refcount_sub_and_test(decrement, &nf->nf_ref))] (with "decrement" being 2), so the nf->nf_ref goes to 0, the 'nf' is added to the dispose list [list_add(&nf->nf_lru, dispose)] * nfsd_file_put() detects NFSD_FILE_HASHED bit is cleared, so it tries to remove the 'nf' from the LRU [if (!nfsd_file_lru_remove(nf))], although the 'nf' is not in the LRU, but it is linked in the 'dispose' list, nfsd_file_lru_remove() simply treats it as part of the LRU and removes it. This leads to its removal from the 'dispose' list! * Now nf->ref is 0, unhashed. nfsd_file_put() continues its execution and set nf->nf_ref to REFCOUNT_SATURATED. As shown in the above analysis, using nf_lru for both the LRU list and dispose list can cause the leaks. This patch adds a new list_head nf_gc in struct nfsd_file, and uses it for the dispose list. This does not fix the nfsd_file leaking issue completely. Signed-off-by: Youzhong Yang Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/filecache.c | 18 ++++++++++-------- fs/nfsd/filecache.h | 1 + 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/fs/nfsd/filecache.c b/fs/nfsd/filecache.c index f4704f5d4086..01d76f6bc80a 100644 --- a/fs/nfsd/filecache.c +++ b/fs/nfsd/filecache.c @@ -216,6 +216,7 @@ nfsd_file_alloc(struct net *net, struct inode *inode, unsigned char need, return NULL; INIT_LIST_HEAD(&nf->nf_lru); + INIT_LIST_HEAD(&nf->nf_gc); nf->nf_birthtime = ktime_get(); nf->nf_file = NULL; nf->nf_cred = get_current_cred(); @@ -393,8 +394,8 @@ nfsd_file_dispose_list(struct list_head *dispose) struct nfsd_file *nf; while (!list_empty(dispose)) { - nf = list_first_entry(dispose, struct nfsd_file, nf_lru); - list_del_init(&nf->nf_lru); + nf = list_first_entry(dispose, struct nfsd_file, nf_gc); + list_del_init(&nf->nf_gc); nfsd_file_free(nf); } } @@ -411,12 +412,12 @@ nfsd_file_dispose_list_delayed(struct list_head *dispose) { while(!list_empty(dispose)) { struct nfsd_file *nf = list_first_entry(dispose, - struct nfsd_file, nf_lru); + struct nfsd_file, nf_gc); struct nfsd_net *nn = net_generic(nf->nf_net, nfsd_net_id); struct nfsd_fcache_disposal *l = nn->fcache_disposal; spin_lock(&l->lock); - list_move_tail(&nf->nf_lru, &l->freeme); + list_move_tail(&nf->nf_gc, &l->freeme); spin_unlock(&l->lock); svc_wake_up(nn->nfsd_serv); } @@ -503,7 +504,8 @@ nfsd_file_lru_cb(struct list_head *item, struct list_lru_one *lru, /* Refcount went to zero. Unhash it and queue it to the dispose list */ nfsd_file_unhash(nf); - list_lru_isolate_move(lru, &nf->nf_lru, head); + list_lru_isolate(lru, &nf->nf_lru); + list_add(&nf->nf_gc, head); this_cpu_inc(nfsd_file_evictions); trace_nfsd_file_gc_disposed(nf); return LRU_REMOVED; @@ -578,7 +580,7 @@ nfsd_file_cond_queue(struct nfsd_file *nf, struct list_head *dispose) /* If refcount goes to 0, then put on the dispose list */ if (refcount_sub_and_test(decrement, &nf->nf_ref)) { - list_add(&nf->nf_lru, dispose); + list_add(&nf->nf_gc, dispose); trace_nfsd_file_closing(nf); } } @@ -654,8 +656,8 @@ nfsd_file_close_inode_sync(struct inode *inode) nfsd_file_queue_for_close(inode, &dispose); while (!list_empty(&dispose)) { - nf = list_first_entry(&dispose, struct nfsd_file, nf_lru); - list_del_init(&nf->nf_lru); + nf = list_first_entry(&dispose, struct nfsd_file, nf_gc); + list_del_init(&nf->nf_gc); nfsd_file_free(nf); } } diff --git a/fs/nfsd/filecache.h b/fs/nfsd/filecache.h index c61884def906..3fbec24eea6c 100644 --- a/fs/nfsd/filecache.h +++ b/fs/nfsd/filecache.h @@ -44,6 +44,7 @@ struct nfsd_file { struct nfsd_file_mark *nf_mark; struct list_head nf_lru; + struct list_head nf_gc; struct rcu_head nf_rcu; ktime_t nf_birthtime; }; From 81a95c2b1d605743220f28db04b8da13a65c4059 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Thu, 11 Jul 2024 15:11:13 -0400 Subject: [PATCH 060/302] nfsd: remove unneeded EEXIST error check in nfsd_do_file_acquire Given that we do the search and insertion while holding the i_lock, I don't think it's possible for us to get EEXIST here. Remove this case. Fixes: c6593366c0bf ("nfsd: don't kill nfsd_files because of lease break error") Signed-off-by: Jeff Layton Tested-by: Youzhong Yang Signed-off-by: Chuck Lever --- fs/nfsd/filecache.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/fs/nfsd/filecache.c b/fs/nfsd/filecache.c index 01d76f6bc80a..9d7bb65b0746 100644 --- a/fs/nfsd/filecache.c +++ b/fs/nfsd/filecache.c @@ -1037,8 +1037,6 @@ retry: if (likely(ret == 0)) goto open_file; - if (ret == -EEXIST) - goto retry; trace_nfsd_file_insert_err(rqstp, inode, may_flags, ret); status = nfserr_jukebox; goto construction_err; From 8a7926176378460e0d91e02b03f0ff20a8709a60 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Wed, 10 Jul 2024 09:05:32 -0400 Subject: [PATCH 061/302] nfsd: fix refcount leak when file is unhashed after being found If we wait_for_construction and find that the file is no longer hashed, and we're going to retry the open, the old nfsd_file reference is currently leaked. Put the reference before retrying. Fixes: c6593366c0bf ("nfsd: don't kill nfsd_files because of lease break error") Signed-off-by: Jeff Layton Tested-by: Youzhong Yang Signed-off-by: Chuck Lever --- fs/nfsd/filecache.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/nfsd/filecache.c b/fs/nfsd/filecache.c index 9d7bb65b0746..6af4e6027227 100644 --- a/fs/nfsd/filecache.c +++ b/fs/nfsd/filecache.c @@ -1051,6 +1051,7 @@ wait_for_construction: status = nfserr_jukebox; goto construction_err; } + nfsd_file_put(nf); open_retry = false; fh_put(fhp); goto retry; From 700bb4ff912f954345286e065ff145753a1d5bbe Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Wed, 10 Jul 2024 09:05:33 -0400 Subject: [PATCH 062/302] nfsd: count nfsd_file allocations We already count the frees (via nfsd_file_releases). Count the allocations as well. Also switch the direct call to nfsd_file_slab_free in nfsd_file_do_acquire to nfsd_file_free, so that the allocs and releases match up. Signed-off-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/filecache.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/fs/nfsd/filecache.c b/fs/nfsd/filecache.c index 6af4e6027227..93e2ffa5eae6 100644 --- a/fs/nfsd/filecache.c +++ b/fs/nfsd/filecache.c @@ -56,6 +56,7 @@ static DEFINE_PER_CPU(unsigned long, nfsd_file_cache_hits); static DEFINE_PER_CPU(unsigned long, nfsd_file_acquisitions); +static DEFINE_PER_CPU(unsigned long, nfsd_file_allocations); static DEFINE_PER_CPU(unsigned long, nfsd_file_releases); static DEFINE_PER_CPU(unsigned long, nfsd_file_total_age); static DEFINE_PER_CPU(unsigned long, nfsd_file_evictions); @@ -215,6 +216,7 @@ nfsd_file_alloc(struct net *net, struct inode *inode, unsigned char need, if (unlikely(!nf)) return NULL; + this_cpu_inc(nfsd_file_allocations); INIT_LIST_HEAD(&nf->nf_lru); INIT_LIST_HEAD(&nf->nf_gc); nf->nf_birthtime = ktime_get(); @@ -911,6 +913,7 @@ nfsd_file_cache_shutdown(void) for_each_possible_cpu(i) { per_cpu(nfsd_file_cache_hits, i) = 0; per_cpu(nfsd_file_acquisitions, i) = 0; + per_cpu(nfsd_file_allocations, i) = 0; per_cpu(nfsd_file_releases, i) = 0; per_cpu(nfsd_file_total_age, i) = 0; per_cpu(nfsd_file_evictions, i) = 0; @@ -1026,7 +1029,7 @@ retry: if (unlikely(nf)) { spin_unlock(&inode->i_lock); rcu_read_unlock(); - nfsd_file_slab_free(&new->nf_rcu); + nfsd_file_free(new); goto wait_for_construction; } nf = new; @@ -1200,7 +1203,7 @@ nfsd_file_acquire_opened(struct svc_rqst *rqstp, struct svc_fh *fhp, */ int nfsd_file_cache_stats_show(struct seq_file *m, void *v) { - unsigned long releases = 0, evictions = 0; + unsigned long allocations = 0, releases = 0, evictions = 0; unsigned long hits = 0, acquisitions = 0; unsigned int i, count = 0, buckets = 0; unsigned long lru = 0, total_age = 0; @@ -1225,6 +1228,7 @@ int nfsd_file_cache_stats_show(struct seq_file *m, void *v) for_each_possible_cpu(i) { hits += per_cpu(nfsd_file_cache_hits, i); acquisitions += per_cpu(nfsd_file_acquisitions, i); + allocations += per_cpu(nfsd_file_allocations, i); releases += per_cpu(nfsd_file_releases, i); total_age += per_cpu(nfsd_file_total_age, i); evictions += per_cpu(nfsd_file_evictions, i); @@ -1235,6 +1239,7 @@ int nfsd_file_cache_stats_show(struct seq_file *m, void *v) seq_printf(m, "lru entries: %lu\n", lru); seq_printf(m, "cache hits: %lu\n", hits); seq_printf(m, "acquisitions: %lu\n", acquisitions); + seq_printf(m, "allocations: %lu\n", allocations); seq_printf(m, "releases: %lu\n", releases); seq_printf(m, "evictions: %lu\n", evictions); if (releases) From 4b84551a35e36bbed48850dc870191a13f0841fd Mon Sep 17 00:00:00 2001 From: Youzhong Yang Date: Thu, 11 Jul 2024 11:51:33 -0400 Subject: [PATCH 063/302] nfsd: use system_unbound_wq for nfsd_file_gc_worker() After many rounds of changes in filecache.c, the fix by commit ce7df055(NFSD: Make the file_delayed_close workqueue UNBOUND) is gone, now we are getting syslog messages like these: [ 1618.186688] workqueue: nfsd_file_gc_worker [nfsd] hogged CPU for >13333us 4 times, consider switching to WQ_UNBOUND [ 1638.661616] workqueue: nfsd_file_gc_worker [nfsd] hogged CPU for >13333us 8 times, consider switching to WQ_UNBOUND [ 1665.284542] workqueue: nfsd_file_gc_worker [nfsd] hogged CPU for >13333us 16 times, consider switching to WQ_UNBOUND [ 1759.491342] workqueue: nfsd_file_gc_worker [nfsd] hogged CPU for >13333us 32 times, consider switching to WQ_UNBOUND [ 3013.012308] workqueue: nfsd_file_gc_worker [nfsd] hogged CPU for >13333us 64 times, consider switching to WQ_UNBOUND [ 3154.172827] workqueue: nfsd_file_gc_worker [nfsd] hogged CPU for >13333us 128 times, consider switching to WQ_UNBOUND [ 3422.461924] workqueue: nfsd_file_gc_worker [nfsd] hogged CPU for >13333us 256 times, consider switching to WQ_UNBOUND [ 3963.152054] workqueue: nfsd_file_gc_worker [nfsd] hogged CPU for >13333us 512 times, consider switching to WQ_UNBOUND Consider use system_unbound_wq instead of system_wq for nfsd_file_gc_worker(). Signed-off-by: Youzhong Yang Signed-off-by: Chuck Lever --- fs/nfsd/filecache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/nfsd/filecache.c b/fs/nfsd/filecache.c index 93e2ffa5eae6..9e9d246f993c 100644 --- a/fs/nfsd/filecache.c +++ b/fs/nfsd/filecache.c @@ -112,7 +112,7 @@ static void nfsd_file_schedule_laundrette(void) { if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags)) - queue_delayed_work(system_wq, &nfsd_filecache_laundrette, + queue_delayed_work(system_unbound_wq, &nfsd_filecache_laundrette, NFSD_LAUNDRETTE_DELAY); } From cef48236dfe55fa266d505e8a497963a7bc5ef2a Mon Sep 17 00:00:00 2001 From: Chen Hanxiao Date: Thu, 18 Jul 2024 15:06:16 +0800 Subject: [PATCH 064/302] NFS: trace: show TIMEDOUT instead of 0x6e __nfs_revalidate_inode may return ETIMEDOUT. print symbol of ETIMEDOUT in nfs trace: before: cat-5191 [005] 119.331127: nfs_revalidate_inode_exit: error=-110 (0x6e) after: cat-1738 [004] 44.365509: nfs_revalidate_inode_exit: error=-110 (TIMEDOUT) Signed-off-by: Chen Hanxiao Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- include/trace/misc/nfs.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/trace/misc/nfs.h b/include/trace/misc/nfs.h index 7b221d51133a..c82233e950ac 100644 --- a/include/trace/misc/nfs.h +++ b/include/trace/misc/nfs.h @@ -51,6 +51,7 @@ TRACE_DEFINE_ENUM(NFSERR_JUKEBOX); { NFSERR_IO, "IO" }, \ { NFSERR_NXIO, "NXIO" }, \ { ECHILD, "CHILD" }, \ + { ETIMEDOUT, "TIMEDOUT" }, \ { NFSERR_EAGAIN, "AGAIN" }, \ { NFSERR_ACCES, "ACCES" }, \ { NFSERR_EXIST, "EXIST" }, \ From 8203ab8a9dbe7b2a060fa6bdbfe2f28cb2f73172 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Wed, 24 Jul 2024 12:55:49 +1000 Subject: [PATCH 065/302] nfsd: don't EXPORT_SYMBOL nfsd4_ssc_init_umount_work() nfsd4_ssc_init_umount_work() is only used in the nfsd module, so there is no need to EXPORT it. Signed-off-by: NeilBrown Signed-off-by: Chuck Lever --- fs/nfsd/nfs4state.c | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index a366fb1c1b9b..5c48915ff61c 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -6271,7 +6271,6 @@ void nfsd4_ssc_init_umount_work(struct nfsd_net *nn) INIT_LIST_HEAD(&nn->nfsd_ssc_mount_list); init_waitqueue_head(&nn->nfsd_ssc_waitq); } -EXPORT_SYMBOL_GPL(nfsd4_ssc_init_umount_work); /* * This is called when nfsd is being shutdown, after all inter_ssc From 4ed9ef32606386efc562bada891d7baa16fc46b4 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Mon, 15 Jul 2024 17:14:14 +1000 Subject: [PATCH 066/302] lockd: discard nlmsvc_timeout nlmsvc_timeout always has the same value as (nlm_timeout * HZ), so use that in the one place that nlmsvc_timeout is used. In truth it *might* not always be the same as nlmsvc_timeout is only set when lockd is started while nlm_timeout can be set at anytime via sysctl. I think this difference it not helpful so removing it is good. Also remove the test for nlm_timout being 0. This is not possible - unless a module parameter is used to set the minimum timeout to 0, and if that happens then it probably should be honoured. Signed-off-by: NeilBrown Signed-off-by: Chuck Lever --- fs/lockd/host.c | 2 +- fs/lockd/svc.c | 7 +------ include/linux/lockd/lockd.h | 2 +- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/fs/lockd/host.c b/fs/lockd/host.c index c11516801784..5e6877c37f73 100644 --- a/fs/lockd/host.c +++ b/fs/lockd/host.c @@ -440,7 +440,7 @@ nlm_bind_host(struct nlm_host *host) if ((clnt = host->h_rpcclnt) != NULL) { nlm_rebind_host(host); } else { - unsigned long increment = nlmsvc_timeout; + unsigned long increment = nlm_timeout * HZ; struct rpc_timeout timeparms = { .to_initval = increment, .to_increment = increment, diff --git a/fs/lockd/svc.c b/fs/lockd/svc.c index ab8042a5b895..71713309967d 100644 --- a/fs/lockd/svc.c +++ b/fs/lockd/svc.c @@ -53,7 +53,6 @@ EXPORT_SYMBOL_GPL(nlmsvc_ops); static DEFINE_MUTEX(nlmsvc_mutex); static unsigned int nlmsvc_users; static struct svc_serv *nlmsvc_serv; -unsigned long nlmsvc_timeout; static void nlmsvc_request_retry(struct timer_list *tl) { @@ -68,7 +67,7 @@ unsigned int lockd_net_id; * and also changed through the sysctl interface. -- Jamie Lokier, Aug 2003 */ static unsigned long nlm_grace_period; -static unsigned long nlm_timeout = LOCKD_DFLT_TIMEO; +unsigned long nlm_timeout = LOCKD_DFLT_TIMEO; static int nlm_udpport, nlm_tcpport; /* RLIM_NOFILE defaults to 1024. That seems like a reasonable default here. */ @@ -333,10 +332,6 @@ static int lockd_get(void) printk(KERN_WARNING "lockd_up: no pid, %d users??\n", nlmsvc_users); - if (!nlm_timeout) - nlm_timeout = LOCKD_DFLT_TIMEO; - nlmsvc_timeout = nlm_timeout * HZ; - serv = svc_create(&nlmsvc_program, LOCKD_BUFSIZE, lockd); if (!serv) { printk(KERN_WARNING "lockd_up: create service failed\n"); diff --git a/include/linux/lockd/lockd.h b/include/linux/lockd/lockd.h index 1b95fe31051f..61c4b9c41904 100644 --- a/include/linux/lockd/lockd.h +++ b/include/linux/lockd/lockd.h @@ -200,7 +200,7 @@ extern const struct svc_procedure nlmsvc_procedures[24]; extern const struct svc_procedure nlmsvc_procedures4[24]; #endif extern int nlmsvc_grace_period; -extern unsigned long nlmsvc_timeout; +extern unsigned long nlm_timeout; extern bool nsm_use_hostnames; extern u32 nsm_local_state; From f2b27e1d72527f94f030b6356b3187576e60885b Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Mon, 15 Jul 2024 17:14:15 +1000 Subject: [PATCH 067/302] SUNRPC: make various functions static, or not exported. Various functions are only used within the sunrpc module, and several are only use in the one file. So clean up: These are marked static, and any EXPORT is removed. svc_rcpb_setup() svc_rqst_alloc() svc_rqst_free() - also moved before first use svc_rpcbind_set_version() svc_drop() - also moved to svc.c These are now not EXPORTed, but are not static. svc_authenticate() svc_sock_update_bufs() Signed-off-by: NeilBrown Signed-off-by: Chuck Lever --- include/linux/sunrpc/svc.h | 9 ------- include/linux/sunrpc/svcauth.h | 1 - include/linux/sunrpc/svcsock.h | 2 -- net/sunrpc/sunrpc.h | 4 +++ net/sunrpc/svc.c | 48 ++++++++++++++++++---------------- net/sunrpc/svc_xprt.c | 9 ------- net/sunrpc/svcauth.c | 1 - net/sunrpc/svcsock.c | 1 - 8 files changed, 29 insertions(+), 46 deletions(-) diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index a7d0406b9ef5..e4fa25fafa97 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -401,17 +401,13 @@ struct svc_procedure { */ int sunrpc_set_pool_mode(const char *val); int sunrpc_get_pool_mode(char *val, size_t size); -int svc_rpcb_setup(struct svc_serv *serv, struct net *net); void svc_rpcb_cleanup(struct svc_serv *serv, struct net *net); int svc_bind(struct svc_serv *serv, struct net *net); struct svc_serv *svc_create(struct svc_program *, unsigned int, int (*threadfn)(void *data)); -struct svc_rqst *svc_rqst_alloc(struct svc_serv *serv, - struct svc_pool *pool, int node); bool svc_rqst_replace_page(struct svc_rqst *rqstp, struct page *page); void svc_rqst_release_pages(struct svc_rqst *rqstp); -void svc_rqst_free(struct svc_rqst *); void svc_exit_thread(struct svc_rqst *); struct svc_serv * svc_create_pooled(struct svc_program *prog, struct svc_stat *stats, @@ -446,11 +442,6 @@ int svc_generic_rpcbind_set(struct net *net, u32 version, int family, unsigned short proto, unsigned short port); -int svc_rpcbind_set_version(struct net *net, - const struct svc_program *progp, - u32 version, int family, - unsigned short proto, - unsigned short port); #define RPC_MAX_ADDRBUFLEN (63U) diff --git a/include/linux/sunrpc/svcauth.h b/include/linux/sunrpc/svcauth.h index 61c455f1e1f5..63cf6fb26dcc 100644 --- a/include/linux/sunrpc/svcauth.h +++ b/include/linux/sunrpc/svcauth.h @@ -151,7 +151,6 @@ struct auth_ops { struct svc_xprt; -extern enum svc_auth_status svc_authenticate(struct svc_rqst *rqstp); extern rpc_authflavor_t svc_auth_flavor(struct svc_rqst *rqstp); extern int svc_authorise(struct svc_rqst *rqstp); extern enum svc_auth_status svc_set_client(struct svc_rqst *rqstp); diff --git a/include/linux/sunrpc/svcsock.h b/include/linux/sunrpc/svcsock.h index 7c78ec6356b9..bf45d9e8492a 100644 --- a/include/linux/sunrpc/svcsock.h +++ b/include/linux/sunrpc/svcsock.h @@ -58,8 +58,6 @@ static inline u32 svc_sock_final_rec(struct svc_sock *svsk) */ void svc_recv(struct svc_rqst *rqstp); void svc_send(struct svc_rqst *rqstp); -void svc_drop(struct svc_rqst *); -void svc_sock_update_bufs(struct svc_serv *serv); int svc_addsock(struct svc_serv *serv, struct net *net, const int fd, char *name_return, const size_t len, const struct cred *cred); diff --git a/net/sunrpc/sunrpc.h b/net/sunrpc/sunrpc.h index d4a362c9e4b3..e3c6e3b63f0b 100644 --- a/net/sunrpc/sunrpc.h +++ b/net/sunrpc/sunrpc.h @@ -36,7 +36,11 @@ static inline int sock_is_loopback(struct sock *sk) return loopback; } +struct svc_serv; +struct svc_rqst; int rpc_clients_notifier_register(void); void rpc_clients_notifier_unregister(void); void auth_domain_cleanup(void); +void svc_sock_update_bufs(struct svc_serv *serv); +enum svc_auth_status svc_authenticate(struct svc_rqst *rqstp); #endif /* _NET_SUNRPC_SUNRPC_H */ diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index 88a59cfa5583..561d20a5316e 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -32,6 +32,7 @@ #include #include "fail.h" +#include "sunrpc.h" #define RPCDBG_FACILITY RPCDBG_SVCDSP @@ -417,7 +418,7 @@ struct svc_pool *svc_pool_for_cpu(struct svc_serv *serv) return &serv->sv_pools[pidx % serv->sv_nrpools]; } -int svc_rpcb_setup(struct svc_serv *serv, struct net *net) +static int svc_rpcb_setup(struct svc_serv *serv, struct net *net) { int err; @@ -429,7 +430,6 @@ int svc_rpcb_setup(struct svc_serv *serv, struct net *net) svc_unregister(serv, net); return 0; } -EXPORT_SYMBOL_GPL(svc_rpcb_setup); void svc_rpcb_cleanup(struct svc_serv *serv, struct net *net) { @@ -664,7 +664,20 @@ svc_release_buffer(struct svc_rqst *rqstp) put_page(rqstp->rq_pages[i]); } -struct svc_rqst * +static void +svc_rqst_free(struct svc_rqst *rqstp) +{ + folio_batch_release(&rqstp->rq_fbatch); + svc_release_buffer(rqstp); + if (rqstp->rq_scratch_page) + put_page(rqstp->rq_scratch_page); + kfree(rqstp->rq_resp); + kfree(rqstp->rq_argp); + kfree(rqstp->rq_auth_data); + kfree_rcu(rqstp, rq_rcu_head); +} + +static struct svc_rqst * svc_rqst_alloc(struct svc_serv *serv, struct svc_pool *pool, int node) { struct svc_rqst *rqstp; @@ -698,7 +711,6 @@ out_enomem: svc_rqst_free(rqstp); return NULL; } -EXPORT_SYMBOL_GPL(svc_rqst_alloc); static struct svc_rqst * svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node) @@ -933,24 +945,6 @@ void svc_rqst_release_pages(struct svc_rqst *rqstp) } } -/* - * Called from a server thread as it's exiting. Caller must hold the "service - * mutex" for the service. - */ -void -svc_rqst_free(struct svc_rqst *rqstp) -{ - folio_batch_release(&rqstp->rq_fbatch); - svc_release_buffer(rqstp); - if (rqstp->rq_scratch_page) - put_page(rqstp->rq_scratch_page); - kfree(rqstp->rq_resp); - kfree(rqstp->rq_argp); - kfree(rqstp->rq_auth_data); - kfree_rcu(rqstp, rq_rcu_head); -} -EXPORT_SYMBOL_GPL(svc_rqst_free); - void svc_exit_thread(struct svc_rqst *rqstp) { @@ -1098,6 +1092,7 @@ static int __svc_register(struct net *net, const char *progname, return error; } +static int svc_rpcbind_set_version(struct net *net, const struct svc_program *progp, u32 version, int family, @@ -1108,7 +1103,6 @@ int svc_rpcbind_set_version(struct net *net, version, family, proto, port); } -EXPORT_SYMBOL_GPL(svc_rpcbind_set_version); int svc_generic_rpcbind_set(struct net *net, const struct svc_program *progp, @@ -1526,6 +1520,14 @@ err_system_err: goto sendit; } +/* + * Drop request + */ +static void svc_drop(struct svc_rqst *rqstp) +{ + trace_svc_drop(rqstp); +} + /** * svc_process - Execute one RPC transaction * @rqstp: RPC transaction context diff --git a/net/sunrpc/svc_xprt.c b/net/sunrpc/svc_xprt.c index d3735ab3e6d1..53ebc719ff5a 100644 --- a/net/sunrpc/svc_xprt.c +++ b/net/sunrpc/svc_xprt.c @@ -905,15 +905,6 @@ void svc_recv(struct svc_rqst *rqstp) } EXPORT_SYMBOL_GPL(svc_recv); -/* - * Drop request - */ -void svc_drop(struct svc_rqst *rqstp) -{ - trace_svc_drop(rqstp); -} -EXPORT_SYMBOL_GPL(svc_drop); - /** * svc_send - Return reply to client * @rqstp: RPC transaction context diff --git a/net/sunrpc/svcauth.c b/net/sunrpc/svcauth.c index 1619211f0960..93d9e949e265 100644 --- a/net/sunrpc/svcauth.c +++ b/net/sunrpc/svcauth.c @@ -98,7 +98,6 @@ enum svc_auth_status svc_authenticate(struct svc_rqst *rqstp) rqstp->rq_authop = aops; return aops->accept(rqstp); } -EXPORT_SYMBOL_GPL(svc_authenticate); /** * svc_set_client - Assign an appropriate 'auth_domain' as the client diff --git a/net/sunrpc/svcsock.c b/net/sunrpc/svcsock.c index 6b3f01beb294..825ec5357691 100644 --- a/net/sunrpc/svcsock.c +++ b/net/sunrpc/svcsock.c @@ -1378,7 +1378,6 @@ void svc_sock_update_bufs(struct svc_serv *serv) set_bit(XPT_CHNGBUF, &svsk->sk_xprt.xpt_flags); spin_unlock_bh(&serv->sv_lock); } -EXPORT_SYMBOL_GPL(svc_sock_update_bufs); /* * Initialize socket for RPC use and create svc_sock struct From c9f10f811cf707e7d7a33e9f7ff678aab9f85551 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Mon, 15 Jul 2024 17:14:16 +1000 Subject: [PATCH 068/302] nfsd: move nfsd_pool_stats_open into nfsctl.c nfsd_pool_stats_open() is used in nfsctl.c, so move it there. Signed-off-by: NeilBrown Signed-off-by: Chuck Lever --- fs/nfsd/nfsctl.c | 7 +++++++ fs/nfsd/nfsd.h | 2 -- fs/nfsd/nfssvc.c | 7 ------- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c index 34eb2c2cbcde..97c8c76b241b 100644 --- a/fs/nfsd/nfsctl.c +++ b/fs/nfsd/nfsctl.c @@ -174,6 +174,13 @@ static int export_features_show(struct seq_file *m, void *v) DEFINE_SHOW_ATTRIBUTE(export_features); +static int nfsd_pool_stats_open(struct inode *inode, struct file *file) +{ + struct nfsd_net *nn = net_generic(inode->i_sb->s_fs_info, nfsd_net_id); + + return svc_pool_stats_open(&nn->nfsd_info, file); +} + static const struct file_operations pool_stats_operations = { .open = nfsd_pool_stats_open, .read = seq_read, diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h index cec8697b1cd6..39e109a7d56d 100644 --- a/fs/nfsd/nfsd.h +++ b/fs/nfsd/nfsd.h @@ -111,8 +111,6 @@ int nfsd_nrthreads(struct net *); int nfsd_nrpools(struct net *); int nfsd_get_nrthreads(int n, int *, struct net *); int nfsd_set_nrthreads(int n, int *, struct net *); -int nfsd_pool_stats_open(struct inode *, struct file *); -int nfsd_pool_stats_release(struct inode *, struct file *); void nfsd_shutdown_threads(struct net *net); bool i_am_nfsd(void); diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c index 0bc8eaa5e009..f25b26bc5670 100644 --- a/fs/nfsd/nfssvc.c +++ b/fs/nfsd/nfssvc.c @@ -1084,10 +1084,3 @@ bool nfssvc_encode_voidres(struct svc_rqst *rqstp, struct xdr_stream *xdr) { return true; } - -int nfsd_pool_stats_open(struct inode *inode, struct file *file) -{ - struct nfsd_net *nn = net_generic(inode->i_sb->s_fs_info, nfsd_net_id); - - return svc_pool_stats_open(&nn->nfsd_info, file); -} From 4654cf52cbd07cb2d6ab6f55bcc5eb2dae8b736a Mon Sep 17 00:00:00 2001 From: Riyan Dhiman Date: Sun, 1 Sep 2024 14:56:02 +0530 Subject: [PATCH 069/302] PCI: vmd: Fix indentation issue in vmd_shutdown() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The code in vmd_shutdown() had an indentation issue where spaces were used instead of tabs. This commit corrects the indentation to use tabs, adhering to the Linux kernel coding style guidelines. Issue reported by the checkpatch.pl script: ERROR: code indent should use tabs where possible #1056: FILE: drivers/pci/controller/vmd.c:1056: + struct vmd_dev *vmd = pci_get_drvdata(dev);$ WARNING: please, no spaces at the start of a line #1056: FILE: drivers/pci/controller/vmd.c:1056: + struct vmd_dev *vmd = pci_get_drvdata(dev);$ ERROR: code indent should use tabs where possible #1058: FILE: drivers/pci/controller/vmd.c:1058: + vmd_remove_irq_domain(vmd);$ WARNING: please, no spaces at the start of a line #1058: FILE: drivers/pci/controller/vmd.c:1058: + vmd_remove_irq_domain(vmd);$ No functional changes are intended. Link: https://lore.kernel.org/linux-pci/20240901092602.17414-1-riyandhiman14@gmail.com Signed-off-by: Riyan Dhiman [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński --- drivers/pci/controller/vmd.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/pci/controller/vmd.c b/drivers/pci/controller/vmd.c index a726de0af011..28ce903b96b6 100644 --- a/drivers/pci/controller/vmd.c +++ b/drivers/pci/controller/vmd.c @@ -1053,9 +1053,9 @@ static void vmd_remove(struct pci_dev *dev) static void vmd_shutdown(struct pci_dev *dev) { - struct vmd_dev *vmd = pci_get_drvdata(dev); + struct vmd_dev *vmd = pci_get_drvdata(dev); - vmd_remove_irq_domain(vmd); + vmd_remove_irq_domain(vmd); } #ifdef CONFIG_PM_SLEEP From 3ee1a6b5d78f07bc026f71ea0664d8df2a7d1280 Mon Sep 17 00:00:00 2001 From: Jinjie Ruan Date: Thu, 29 Aug 2024 10:24:35 +0800 Subject: [PATCH 070/302] PCI/AER: Use PCI_DEVID() macro in aer_inject() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The PCI_DEVID() macro can be used instead of open-coding it. No functional changes intended. Link: https://lore.kernel.org/linux-pci/20240829022435.4145181-1-ruanjinjie@huawei.com Signed-off-by: Jinjie Ruan [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński --- drivers/pci/pcie/aer_inject.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/pci/pcie/aer_inject.c b/drivers/pci/pcie/aer_inject.c index f81b2303bf6a..91acc7b17f68 100644 --- a/drivers/pci/pcie/aer_inject.c +++ b/drivers/pci/pcie/aer_inject.c @@ -430,7 +430,7 @@ static int aer_inject(struct aer_error_inj *einj) else rperr->root_status |= PCI_ERR_ROOT_COR_RCV; rperr->source_id &= 0xffff0000; - rperr->source_id |= (einj->bus << 8) | devfn; + rperr->source_id |= PCI_DEVID(einj->bus, devfn); } if (einj->uncor_status) { if (rperr->root_status & PCI_ERR_ROOT_UNCOR_RCV) @@ -443,7 +443,7 @@ static int aer_inject(struct aer_error_inj *einj) rperr->root_status |= PCI_ERR_ROOT_NONFATAL_RCV; rperr->root_status |= PCI_ERR_ROOT_UNCOR_RCV; rperr->source_id &= 0x0000ffff; - rperr->source_id |= ((einj->bus << 8) | devfn) << 16; + rperr->source_id |= PCI_DEVID(einj->bus, devfn) << 16; } spin_unlock_irqrestore(&inject_lock, flags); From 5603a3491b368faf180472f4bdc6480c13c7385e Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Thu, 22 Aug 2024 21:59:41 +0100 Subject: [PATCH 071/302] PCI: rcar-gen4: Make read-only const array check_addr static MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Don't populate the const read-only array check_addr on the stack at run time, instead make it static. Link: https://lore.kernel.org/linux-pci/20240822205941.643187-1-colin.i.king@gmail.com Signed-off-by: Colin Ian King [kwilczynski: refactor array definition] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Geert Uytterhoeven --- drivers/pci/controller/dwc/pcie-rcar-gen4.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/pci/controller/dwc/pcie-rcar-gen4.c b/drivers/pci/controller/dwc/pcie-rcar-gen4.c index f0f3ebd1a033..7d18853a46de 100644 --- a/drivers/pci/controller/dwc/pcie-rcar-gen4.c +++ b/drivers/pci/controller/dwc/pcie-rcar-gen4.c @@ -606,7 +606,12 @@ static int rcar_gen4_pcie_reg_test_bit(struct rcar_gen4_pcie *rcar, static int rcar_gen4_pcie_download_phy_firmware(struct rcar_gen4_pcie *rcar) { /* The check_addr values are magical numbers in the datasheet */ - const u32 check_addr[] = { 0x00101018, 0x00101118, 0x00101021, 0x00101121}; + static const u32 check_addr[] = { + 0x00101018, + 0x00101118, + 0x00101021, + 0x00101121, + }; struct dw_pcie *dw = &rcar->dw; const struct firmware *fw; unsigned int i, timeout; From 901849e7070b2b74ba1adaaf025a09a1f381fefe Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 19 Jul 2024 18:51:01 +0100 Subject: [PATCH 072/302] gfs2: Add gfs2_aspace_writepages() This saves one indirect function call per folio and gets us closer to removing aops->writepage. Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andreas Gruenbacher --- fs/gfs2/meta_io.c | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/fs/gfs2/meta_io.c b/fs/gfs2/meta_io.c index 2b26e8d529aa..cfb204c9396c 100644 --- a/fs/gfs2/meta_io.c +++ b/fs/gfs2/meta_io.c @@ -30,9 +30,9 @@ #include "util.h" #include "trace_gfs2.h" -static int gfs2_aspace_writepage(struct page *page, struct writeback_control *wbc) +static void gfs2_aspace_write_folio(struct folio *folio, + struct writeback_control *wbc) { - struct folio *folio = page_folio(page); struct buffer_head *bh, *head; int nr_underway = 0; blk_opf_t write_flags = REQ_META | REQ_PRIO | wbc_to_write_flags(wbc); @@ -66,8 +66,8 @@ static int gfs2_aspace_writepage(struct page *page, struct writeback_control *wb } while ((bh = bh->b_this_page) != head); /* - * The page and its buffers are protected by PageWriteback(), so we can - * drop the bh refcounts early. + * The folio and its buffers are protected from truncation by + * the writeback flag, so we can drop the bh refcounts early. */ BUG_ON(folio_test_writeback(folio)); folio_start_writeback(folio); @@ -84,14 +84,31 @@ static int gfs2_aspace_writepage(struct page *page, struct writeback_control *wb if (nr_underway == 0) folio_end_writeback(folio); +} + +static int gfs2_aspace_writepage(struct page *page, struct writeback_control *wbc) +{ + gfs2_aspace_write_folio(page_folio(page), wbc); return 0; } +static int gfs2_aspace_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + struct folio *folio = NULL; + int error; + + while ((folio = writeback_iter(mapping, wbc, folio, &error))) + gfs2_aspace_write_folio(folio, wbc); + + return error; +} + const struct address_space_operations gfs2_meta_aops = { .dirty_folio = block_dirty_folio, .invalidate_folio = block_invalidate_folio, - .writepage = gfs2_aspace_writepage, + .writepages = gfs2_aspace_writepages, .release_folio = gfs2_release_folio, }; @@ -99,6 +116,7 @@ const struct address_space_operations gfs2_rgrp_aops = { .dirty_folio = block_dirty_folio, .invalidate_folio = block_invalidate_folio, .writepage = gfs2_aspace_writepage, + .writepages = gfs2_aspace_writepages, .release_folio = gfs2_release_folio, }; From 8d391972ae2d4fcee9c928581dfb6cb4ce6a3938 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 19 Jul 2024 18:51:02 +0100 Subject: [PATCH 073/302] gfs2: Remove __gfs2_writepage() Call aops->writepages() instead of using write_cache_pages() to call aops->writepage. Change the handling of -ENODATA to not set the persistent error on the block device. Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andreas Gruenbacher --- fs/gfs2/log.c | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c index 6ee6013fb825..f9c5089783d2 100644 --- a/fs/gfs2/log.c +++ b/fs/gfs2/log.c @@ -80,15 +80,6 @@ void gfs2_remove_from_ail(struct gfs2_bufdata *bd) brelse(bd->bd_bh); } -static int __gfs2_writepage(struct folio *folio, struct writeback_control *wbc, - void *data) -{ - struct address_space *mapping = data; - int ret = mapping->a_ops->writepage(&folio->page, wbc); - mapping_set_error(mapping, ret); - return ret; -} - /** * gfs2_ail1_start_one - Start I/O on a transaction * @sdp: The superblock @@ -140,7 +131,7 @@ __acquires(&sdp->sd_ail_lock) if (!mapping) continue; spin_unlock(&sdp->sd_ail_lock); - ret = write_cache_pages(mapping, wbc, __gfs2_writepage, mapping); + ret = mapping->a_ops->writepages(mapping, wbc); if (need_resched()) { blk_finish_plug(plug); cond_resched(); @@ -149,6 +140,7 @@ __acquires(&sdp->sd_ail_lock) spin_lock(&sdp->sd_ail_lock); if (ret == -ENODATA) /* if a jdata write into a new hole */ ret = 0; /* ignore it */ + mapping_set_error(mapping, ret); if (ret || wbc->nr_to_write <= 0) break; return -EBUSY; From e5ac17199275faffc5399ff7c8a0bea1db553f26 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 19 Jul 2024 18:51:03 +0100 Subject: [PATCH 074/302] gfs2: Remove gfs2_jdata_writepage() There are no remaining callers of gfs2_jdata_writepage() other than vmscan, which is known to do more harm than good. Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andreas Gruenbacher --- fs/gfs2/aops.c | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/fs/gfs2/aops.c b/fs/gfs2/aops.c index 10d5acd3f742..68fc8af14700 100644 --- a/fs/gfs2/aops.c +++ b/fs/gfs2/aops.c @@ -138,35 +138,6 @@ static int __gfs2_jdata_write_folio(struct folio *folio, return gfs2_write_jdata_folio(folio, wbc); } -/** - * gfs2_jdata_writepage - Write complete page - * @page: Page to write - * @wbc: The writeback control - * - * Returns: errno - * - */ - -static int gfs2_jdata_writepage(struct page *page, struct writeback_control *wbc) -{ - struct folio *folio = page_folio(page); - struct inode *inode = page->mapping->host; - struct gfs2_inode *ip = GFS2_I(inode); - struct gfs2_sbd *sdp = GFS2_SB(inode); - - if (gfs2_assert_withdraw(sdp, ip->i_gl->gl_state == LM_ST_EXCLUSIVE)) - goto out; - if (folio_test_checked(folio) || current->journal_info) - goto out_ignore; - return __gfs2_jdata_write_folio(folio, wbc); - -out_ignore: - folio_redirty_for_writepage(wbc, folio); -out: - folio_unlock(folio); - return 0; -} - /** * gfs2_writepages - Write a bunch of dirty pages back to disk * @mapping: The mapping to write @@ -748,7 +719,6 @@ static const struct address_space_operations gfs2_aops = { }; static const struct address_space_operations gfs2_jdata_aops = { - .writepage = gfs2_jdata_writepage, .writepages = gfs2_jdata_writepages, .read_folio = gfs2_read_folio, .readahead = gfs2_readahead, From 6888c1e85f5db129e6ddcff879bb127bbfdb5c64 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 19 Jul 2024 18:51:04 +0100 Subject: [PATCH 075/302] gfs2: Remove gfs2_aspace_writepage() There are no remaining callers of gfs2_aspace_writepage() other than vmscan, which is known to do more harm than good. Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andreas Gruenbacher --- fs/gfs2/meta_io.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/fs/gfs2/meta_io.c b/fs/gfs2/meta_io.c index cfb204c9396c..fea3efcc2f93 100644 --- a/fs/gfs2/meta_io.c +++ b/fs/gfs2/meta_io.c @@ -86,13 +86,6 @@ static void gfs2_aspace_write_folio(struct folio *folio, folio_end_writeback(folio); } -static int gfs2_aspace_writepage(struct page *page, struct writeback_control *wbc) -{ - gfs2_aspace_write_folio(page_folio(page), wbc); - - return 0; -} - static int gfs2_aspace_writepages(struct address_space *mapping, struct writeback_control *wbc) { @@ -115,7 +108,6 @@ const struct address_space_operations gfs2_meta_aops = { const struct address_space_operations gfs2_rgrp_aops = { .dirty_folio = block_dirty_folio, .invalidate_folio = block_invalidate_folio, - .writepage = gfs2_aspace_writepage, .writepages = gfs2_aspace_writepages, .release_folio = gfs2_release_folio, }; From 116249b12939a8ec13eb50f36b6fffd1c719a9ed Mon Sep 17 00:00:00 2001 From: Thorsten Blum Date: Fri, 30 Aug 2024 18:49:03 +0200 Subject: [PATCH 076/302] isofs: Annotate struct SL_component with __counted_by() Add the __counted_by compiler attribute to the flexible array member text to improve access bounds-checking via CONFIG_UBSAN_BOUNDS and CONFIG_FORTIFY_SOURCE. Signed-off-by: Thorsten Blum Signed-off-by: Jan Kara Link: https://patch.msgid.link/20240830164902.112682-2-thorsten.blum@toblux.com --- fs/isofs/rock.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/isofs/rock.h b/fs/isofs/rock.h index ee9660e9671c..7755e587f778 100644 --- a/fs/isofs/rock.h +++ b/fs/isofs/rock.h @@ -44,7 +44,7 @@ struct RR_PN_s { struct SL_component { __u8 flags; __u8 len; - __u8 text[]; + __u8 text[] __counted_by(len); } __attribute__ ((packed)); struct RR_SL_s { From c6abd0eadec65a929bd69718daae887197f85dd3 Mon Sep 17 00:00:00 2001 From: Lorenzo Bianconi Date: Wed, 3 Jul 2024 18:12:41 +0200 Subject: [PATCH 077/302] dt-bindings: PCI: mediatek-gen3: Add support for Airoha EN7581 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce Airoha EN7581 entry in mediatek-gen3 PCIe controller binding. Link: https://lore.kernel.org/linux-pci/138d65a140c3dcf2a6aefecc33ba6ba3ca300a23.1720022580.git.lorenzo@kernel.org Signed-off-by: Lorenzo Bianconi Signed-off-by: Krzysztof Wilczyński Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Conor Dooley Acked-by: Jianjun Wang --- .../bindings/pci/mediatek-pcie-gen3.yaml | 68 +++++++++++++++++-- 1 file changed, 63 insertions(+), 5 deletions(-) diff --git a/Documentation/devicetree/bindings/pci/mediatek-pcie-gen3.yaml b/Documentation/devicetree/bindings/pci/mediatek-pcie-gen3.yaml index 76d742051f73..898c1be2d6a4 100644 --- a/Documentation/devicetree/bindings/pci/mediatek-pcie-gen3.yaml +++ b/Documentation/devicetree/bindings/pci/mediatek-pcie-gen3.yaml @@ -53,6 +53,7 @@ properties: - mediatek,mt8195-pcie - const: mediatek,mt8192-pcie - const: mediatek,mt8192-pcie + - const: airoha,en7581-pcie reg: maxItems: 1 @@ -76,20 +77,20 @@ properties: resets: minItems: 1 - maxItems: 2 + maxItems: 3 reset-names: minItems: 1 - maxItems: 2 + maxItems: 3 items: - enum: [ phy, mac ] + enum: [ phy, mac, phy-lane0, phy-lane1, phy-lane2 ] clocks: - minItems: 4 + minItems: 1 maxItems: 6 clock-names: - minItems: 4 + minItems: 1 maxItems: 6 assigned-clocks: @@ -147,6 +148,9 @@ allOf: const: mediatek,mt8192-pcie then: properties: + clocks: + minItems: 4 + clock-names: items: - const: pl_250m @@ -155,6 +159,15 @@ allOf: - const: tl_32k - const: peri_26m - const: top_133m + + resets: + minItems: 1 + maxItems: 2 + + reset-names: + minItems: 1 + maxItems: 2 + - if: properties: compatible: @@ -164,6 +177,9 @@ allOf: - mediatek,mt8195-pcie then: properties: + clocks: + minItems: 4 + clock-names: items: - const: pl_250m @@ -172,6 +188,15 @@ allOf: - const: tl_32k - const: peri_26m - const: peri_mem + + resets: + minItems: 1 + maxItems: 2 + + reset-names: + minItems: 1 + maxItems: 2 + - if: properties: compatible: @@ -180,6 +205,9 @@ allOf: - mediatek,mt7986-pcie then: properties: + clocks: + minItems: 4 + clock-names: items: - const: pl_250m @@ -187,6 +215,36 @@ allOf: - const: peri_26m - const: top_133m + resets: + minItems: 1 + maxItems: 2 + + reset-names: + minItems: 1 + maxItems: 2 + + - if: + properties: + compatible: + const: airoha,en7581-pcie + then: + properties: + clocks: + maxItems: 1 + + clock-names: + items: + - const: sys-ck + + resets: + minItems: 3 + + reset-names: + items: + - const: phy-lane0 + - const: phy-lane1 + - const: phy-lane2 + unevaluatedProperties: false examples: From dc869a40d73ee6e9f47d683690ae507e30e56044 Mon Sep 17 00:00:00 2001 From: Lorenzo Bianconi Date: Wed, 3 Jul 2024 18:12:42 +0200 Subject: [PATCH 078/302] PCI: mediatek-gen3: Add mtk_gen3_pcie_pdata data structure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce mtk_gen3_pcie_pdata data structure in order to define multiple callbacks for each supported SoC. This is a preliminary patch to introduce EN7581 PCIe support. Link: https://lore.kernel.org/linux-pci/c193d1a87505d045e2e0ef33317bce17012ee095.1720022580.git.lorenzo@kernel.org Signed-off-by: Lorenzo Bianconi Signed-off-by: Krzysztof Wilczyński Tested-by: Zhengping Zhang Reviewed-by: AngeloGioacchino Del Regno Acked-by: Jianjun Wang --- drivers/pci/controller/pcie-mediatek-gen3.c | 24 ++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/drivers/pci/controller/pcie-mediatek-gen3.c b/drivers/pci/controller/pcie-mediatek-gen3.c index b7e8e24f6a40..9ac55771aba8 100644 --- a/drivers/pci/controller/pcie-mediatek-gen3.c +++ b/drivers/pci/controller/pcie-mediatek-gen3.c @@ -100,6 +100,16 @@ #define PCIE_ATR_TLP_TYPE_MEM PCIE_ATR_TLP_TYPE(0) #define PCIE_ATR_TLP_TYPE_IO PCIE_ATR_TLP_TYPE(2) +struct mtk_gen3_pcie; + +/** + * struct mtk_gen3_pcie_pdata - differentiate between host generations + * @power_up: pcie power_up callback + */ +struct mtk_gen3_pcie_pdata { + int (*power_up)(struct mtk_gen3_pcie *pcie); +}; + /** * struct mtk_msi_set - MSI information for each set * @base: IO mapped register base @@ -131,6 +141,7 @@ struct mtk_msi_set { * @msi_sets: MSI sets information * @lock: lock protecting IRQ bit map * @msi_irq_in_use: bit map for assigned MSI IRQ + * @soc: pointer to SoC-dependent operations */ struct mtk_gen3_pcie { struct device *dev; @@ -151,6 +162,8 @@ struct mtk_gen3_pcie { struct mtk_msi_set msi_sets[PCIE_MSI_SET_NUM]; struct mutex lock; DECLARE_BITMAP(msi_irq_in_use, PCIE_MSI_IRQS_NUM); + + const struct mtk_gen3_pcie_pdata *soc; }; /* LTSSM state in PCIE_LTSSM_STATUS_REG bit[28:24] */ @@ -904,7 +917,7 @@ static int mtk_pcie_setup(struct mtk_gen3_pcie *pcie) usleep_range(10, 20); /* Don't touch the hardware registers before power up */ - err = mtk_pcie_power_up(pcie); + err = pcie->soc->power_up(pcie); if (err) return err; @@ -939,6 +952,7 @@ static int mtk_pcie_probe(struct platform_device *pdev) pcie = pci_host_bridge_priv(host); pcie->dev = dev; + pcie->soc = device_get_match_data(dev); platform_set_drvdata(pdev, pcie); err = mtk_pcie_setup(pcie); @@ -1054,7 +1068,7 @@ static int mtk_pcie_resume_noirq(struct device *dev) struct mtk_gen3_pcie *pcie = dev_get_drvdata(dev); int err; - err = mtk_pcie_power_up(pcie); + err = pcie->soc->power_up(pcie); if (err) return err; @@ -1074,8 +1088,12 @@ static const struct dev_pm_ops mtk_pcie_pm_ops = { mtk_pcie_resume_noirq) }; +static const struct mtk_gen3_pcie_pdata mtk_pcie_soc_mt8192 = { + .power_up = mtk_pcie_power_up, +}; + static const struct of_device_id mtk_pcie_of_match[] = { - { .compatible = "mediatek,mt8192-pcie" }, + { .compatible = "mediatek,mt8192-pcie", .data = &mtk_pcie_soc_mt8192 }, {}, }; MODULE_DEVICE_TABLE(of, mtk_pcie_of_match); From ee9eabbe3f0f0c7458d89840add97e54d4e0bccf Mon Sep 17 00:00:00 2001 From: Lorenzo Bianconi Date: Wed, 3 Jul 2024 18:12:43 +0200 Subject: [PATCH 079/302] PCI: mediatek-gen3: Rely on reset_bulk APIs for PHY reset lines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use reset_bulk APIs to manage PHY reset lines. This is a preliminary patch in order to add Airoha EN7581 PCIe support. Link: https://lore.kernel.org/linux-pci/3ceb83bc0defbcf868521f8df4b9100e55ec2614.1720022580.git.lorenzo@kernel.org Signed-off-by: Lorenzo Bianconi Signed-off-by: Krzysztof Wilczyński Tested-by: Zhengping Zhang Reviewed-by: AngeloGioacchino Del Regno Acked-by: Jianjun Wang --- drivers/pci/controller/pcie-mediatek-gen3.c | 45 +++++++++++++++------ 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/drivers/pci/controller/pcie-mediatek-gen3.c b/drivers/pci/controller/pcie-mediatek-gen3.c index 9ac55771aba8..c325460d65a3 100644 --- a/drivers/pci/controller/pcie-mediatek-gen3.c +++ b/drivers/pci/controller/pcie-mediatek-gen3.c @@ -100,14 +100,21 @@ #define PCIE_ATR_TLP_TYPE_MEM PCIE_ATR_TLP_TYPE(0) #define PCIE_ATR_TLP_TYPE_IO PCIE_ATR_TLP_TYPE(2) +#define MAX_NUM_PHY_RESETS 1 + struct mtk_gen3_pcie; /** * struct mtk_gen3_pcie_pdata - differentiate between host generations * @power_up: pcie power_up callback + * @phy_resets: phy reset lines SoC data. */ struct mtk_gen3_pcie_pdata { int (*power_up)(struct mtk_gen3_pcie *pcie); + struct { + const char *id[MAX_NUM_PHY_RESETS]; + int num_resets; + } phy_resets; }; /** @@ -128,7 +135,7 @@ struct mtk_msi_set { * @base: IO mapped register base * @reg_base: physical register base * @mac_reset: MAC reset control - * @phy_reset: PHY reset control + * @phy_resets: PHY reset controllers * @phy: PHY controller block * @clks: PCIe clocks * @num_clks: PCIe clocks count for this port @@ -148,7 +155,7 @@ struct mtk_gen3_pcie { void __iomem *base; phys_addr_t reg_base; struct reset_control *mac_reset; - struct reset_control *phy_reset; + struct reset_control_bulk_data phy_resets[MAX_NUM_PHY_RESETS]; struct phy *phy; struct clk_bulk_data *clks; int num_clks; @@ -788,10 +795,10 @@ static int mtk_pcie_setup_irq(struct mtk_gen3_pcie *pcie) static int mtk_pcie_parse_port(struct mtk_gen3_pcie *pcie) { + int i, ret, num_resets = pcie->soc->phy_resets.num_resets; struct device *dev = pcie->dev; struct platform_device *pdev = to_platform_device(dev); struct resource *regs; - int ret; regs = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pcie-mac"); if (!regs) @@ -804,12 +811,12 @@ static int mtk_pcie_parse_port(struct mtk_gen3_pcie *pcie) pcie->reg_base = regs->start; - pcie->phy_reset = devm_reset_control_get_optional_exclusive(dev, "phy"); - if (IS_ERR(pcie->phy_reset)) { - ret = PTR_ERR(pcie->phy_reset); - if (ret != -EPROBE_DEFER) - dev_err(dev, "failed to get PHY reset\n"); + for (i = 0; i < num_resets; i++) + pcie->phy_resets[i].id = pcie->soc->phy_resets.id[i]; + ret = devm_reset_control_bulk_get_optional_shared(dev, num_resets, pcie->phy_resets); + if (ret) { + dev_err(dev, "failed to get PHY bulk reset\n"); return ret; } @@ -846,7 +853,11 @@ static int mtk_pcie_power_up(struct mtk_gen3_pcie *pcie) int err; /* PHY power on and enable pipe clock */ - reset_control_deassert(pcie->phy_reset); + err = reset_control_bulk_deassert(pcie->soc->phy_resets.num_resets, pcie->phy_resets); + if (err) { + dev_err(dev, "failed to deassert PHYs\n"); + return err; + } err = phy_init(pcie->phy); if (err) { @@ -882,7 +893,7 @@ err_clk_init: err_phy_on: phy_exit(pcie->phy); err_phy_init: - reset_control_assert(pcie->phy_reset); + reset_control_bulk_assert(pcie->soc->phy_resets.num_resets, pcie->phy_resets); return err; } @@ -897,7 +908,7 @@ static void mtk_pcie_power_down(struct mtk_gen3_pcie *pcie) phy_power_off(pcie->phy); phy_exit(pcie->phy); - reset_control_assert(pcie->phy_reset); + reset_control_bulk_assert(pcie->soc->phy_resets.num_resets, pcie->phy_resets); } static int mtk_pcie_setup(struct mtk_gen3_pcie *pcie) @@ -908,11 +919,17 @@ static int mtk_pcie_setup(struct mtk_gen3_pcie *pcie) if (err) return err; + /* + * Deassert the line in order to avoid unbalance in deassert_count + * counter since the bulk is shared. + */ + reset_control_bulk_deassert(pcie->soc->phy_resets.num_resets, pcie->phy_resets); /* * The controller may have been left out of reset by the bootloader * so make sure that we get a clean start by asserting resets here. */ - reset_control_assert(pcie->phy_reset); + reset_control_bulk_assert(pcie->soc->phy_resets.num_resets, pcie->phy_resets); + reset_control_assert(pcie->mac_reset); usleep_range(10, 20); @@ -1090,6 +1107,10 @@ static const struct dev_pm_ops mtk_pcie_pm_ops = { static const struct mtk_gen3_pcie_pdata mtk_pcie_soc_mt8192 = { .power_up = mtk_pcie_power_up, + .phy_resets = { + .id[0] = "phy", + .num_resets = 1, + }, }; static const struct of_device_id mtk_pcie_of_match[] = { From f6ab898356dd70f267c49045a79d28ea5cf5e43e Mon Sep 17 00:00:00 2001 From: Lorenzo Bianconi Date: Wed, 3 Jul 2024 18:12:44 +0200 Subject: [PATCH 080/302] PCI: mediatek-gen3: Add Airoha EN7581 support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce support for Airoha EN7581 PCIe controller to mediatek-gen3 PCIe controller driver. Link: https://lore.kernel.org/linux-pci/aca00bd672ee576ad96d279414fc0835ff31f637.1720022580.git.lorenzo@kernel.org Signed-off-by: Lorenzo Bianconi Signed-off-by: Krzysztof Wilczyński Tested-by: Zhengping Zhang Reviewed-by: AngeloGioacchino Del Regno Acked-by: Jianjun Wang --- drivers/pci/controller/Kconfig | 2 +- drivers/pci/controller/pcie-mediatek-gen3.c | 113 +++++++++++++++++++- 2 files changed, 113 insertions(+), 2 deletions(-) diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index 4d2c188f5835..9800b7681054 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -196,7 +196,7 @@ config PCIE_MEDIATEK config PCIE_MEDIATEK_GEN3 tristate "MediaTek Gen3 PCIe controller" - depends on ARCH_MEDIATEK || COMPILE_TEST + depends on ARCH_AIROHA || ARCH_MEDIATEK || COMPILE_TEST depends on PCI_MSI help Adds support for PCIe Gen3 MAC controller for MediaTek SoCs. diff --git a/drivers/pci/controller/pcie-mediatek-gen3.c b/drivers/pci/controller/pcie-mediatek-gen3.c index c325460d65a3..5c19abac74e8 100644 --- a/drivers/pci/controller/pcie-mediatek-gen3.c +++ b/drivers/pci/controller/pcie-mediatek-gen3.c @@ -6,7 +6,9 @@ * Author: Jianjun Wang */ +#include #include +#include #include #include #include @@ -15,6 +17,8 @@ #include #include #include +#include +#include #include #include #include @@ -29,6 +33,12 @@ #define PCI_CLASS(class) (class << 8) #define PCIE_RC_MODE BIT(0) +#define PCIE_EQ_PRESET_01_REG 0x100 +#define PCIE_VAL_LN0_DOWNSTREAM GENMASK(6, 0) +#define PCIE_VAL_LN0_UPSTREAM GENMASK(14, 8) +#define PCIE_VAL_LN1_DOWNSTREAM GENMASK(22, 16) +#define PCIE_VAL_LN1_UPSTREAM GENMASK(30, 24) + #define PCIE_CFGNUM_REG 0x140 #define PCIE_CFG_DEVFN(devfn) ((devfn) & GENMASK(7, 0)) #define PCIE_CFG_BUS(bus) (((bus) << 8) & GENMASK(15, 8)) @@ -68,6 +78,14 @@ #define PCIE_MSI_SET_ENABLE_REG 0x190 #define PCIE_MSI_SET_ENABLE GENMASK(PCIE_MSI_SET_NUM - 1, 0) +#define PCIE_PIPE4_PIE8_REG 0x338 +#define PCIE_K_FINETUNE_MAX GENMASK(5, 0) +#define PCIE_K_FINETUNE_ERR GENMASK(7, 6) +#define PCIE_K_PRESET_TO_USE GENMASK(18, 8) +#define PCIE_K_PHYPARAM_QUERY BIT(19) +#define PCIE_K_QUERY_TIMEOUT BIT(20) +#define PCIE_K_PRESET_TO_USE_16G GENMASK(31, 21) + #define PCIE_MSI_SET_BASE_REG 0xc00 #define PCIE_MSI_SET_OFFSET 0x10 #define PCIE_MSI_SET_STATUS_OFFSET 0x04 @@ -100,7 +118,10 @@ #define PCIE_ATR_TLP_TYPE_MEM PCIE_ATR_TLP_TYPE(0) #define PCIE_ATR_TLP_TYPE_IO PCIE_ATR_TLP_TYPE(2) -#define MAX_NUM_PHY_RESETS 1 +#define MAX_NUM_PHY_RESETS 3 + +/* Time in ms needed to complete PCIe reset on EN7581 SoC */ +#define PCIE_EN7581_RESET_TIME_MS 100 struct mtk_gen3_pcie; @@ -847,6 +868,85 @@ static int mtk_pcie_parse_port(struct mtk_gen3_pcie *pcie) return 0; } +static int mtk_pcie_en7581_power_up(struct mtk_gen3_pcie *pcie) +{ + struct device *dev = pcie->dev; + int err; + u32 val; + + /* + * Wait for the time needed to complete the bulk assert in + * mtk_pcie_setup for EN7581 SoC. + */ + mdelay(PCIE_EN7581_RESET_TIME_MS); + + err = phy_init(pcie->phy); + if (err) { + dev_err(dev, "failed to initialize PHY\n"); + return err; + } + + err = phy_power_on(pcie->phy); + if (err) { + dev_err(dev, "failed to power on PHY\n"); + goto err_phy_on; + } + + err = reset_control_bulk_deassert(pcie->soc->phy_resets.num_resets, pcie->phy_resets); + if (err) { + dev_err(dev, "failed to deassert PHYs\n"); + goto err_phy_deassert; + } + + /* + * Wait for the time needed to complete the bulk de-assert above. + * This time is specific for EN7581 SoC. + */ + mdelay(PCIE_EN7581_RESET_TIME_MS); + + pm_runtime_enable(dev); + pm_runtime_get_sync(dev); + + err = clk_bulk_prepare(pcie->num_clks, pcie->clks); + if (err) { + dev_err(dev, "failed to prepare clock\n"); + goto err_clk_prepare; + } + + val = FIELD_PREP(PCIE_VAL_LN0_DOWNSTREAM, 0x47) | + FIELD_PREP(PCIE_VAL_LN1_DOWNSTREAM, 0x47) | + FIELD_PREP(PCIE_VAL_LN0_UPSTREAM, 0x41) | + FIELD_PREP(PCIE_VAL_LN1_UPSTREAM, 0x41); + writel_relaxed(val, pcie->base + PCIE_EQ_PRESET_01_REG); + + val = PCIE_K_PHYPARAM_QUERY | PCIE_K_QUERY_TIMEOUT | + FIELD_PREP(PCIE_K_PRESET_TO_USE_16G, 0x80) | + FIELD_PREP(PCIE_K_PRESET_TO_USE, 0x2) | + FIELD_PREP(PCIE_K_FINETUNE_MAX, 0xf); + writel_relaxed(val, pcie->base + PCIE_PIPE4_PIE8_REG); + + err = clk_bulk_enable(pcie->num_clks, pcie->clks); + if (err) { + dev_err(dev, "failed to prepare clock\n"); + goto err_clk_enable; + } + + return 0; + +err_clk_enable: + clk_bulk_unprepare(pcie->num_clks, pcie->clks); +err_clk_prepare: + pm_runtime_put_sync(dev); + pm_runtime_disable(dev); + reset_control_bulk_assert(pcie->soc->phy_resets.num_resets, pcie->phy_resets); +err_phy_deassert: + phy_power_off(pcie->phy); +err_phy_on: + phy_exit(pcie->phy); + + return err; +} + static int mtk_pcie_power_up(struct mtk_gen3_pcie *pcie) { struct device *dev = pcie->dev; @@ -1113,7 +1213,18 @@ static const struct mtk_gen3_pcie_pdata mtk_pcie_soc_mt8192 = { }, }; +static const struct mtk_gen3_pcie_pdata mtk_pcie_soc_en7581 = { + .power_up = mtk_pcie_en7581_power_up, + .phy_resets = { + .id[0] = "phy-lane0", + .id[1] = "phy-lane1", + .id[2] = "phy-lane2", + .num_resets = 3, + }, +}; + static const struct of_device_id mtk_pcie_of_match[] = { + { .compatible = "airoha,en7581-pcie", .data = &mtk_pcie_soc_en7581 }, { .compatible = "mediatek,mt8192-pcie", .data = &mtk_pcie_soc_mt8192 }, {}, }; From d1b6f2e2ce4d8b17d9f3558c98a1517b864bfd03 Mon Sep 17 00:00:00 2001 From: Thomas Richard Date: Wed, 19 Jun 2024 12:15:09 +0200 Subject: [PATCH 081/302] PCI: cadence: Extract link setup sequence from cdns_pcie_host_setup() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The function cdns_pcie_host_setup() mixes probe structure and link setup. The link setup must be done during the resume sequence. So extract it from cdns_pcie_host_setup() and create a dedicated function. Link: https://lore.kernel.org/linux-pci/20240102-j7200-pcie-s2r-v7-1-a2f9156da6c3@bootlin.com Signed-off-by: Thomas Richard Signed-off-by: Krzysztof Wilczyński Reviewed-by: Siddharth Vadapalli --- .../controller/cadence/pcie-cadence-host.c | 39 ++++++++++++------- drivers/pci/controller/cadence/pcie-cadence.h | 6 +++ 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/drivers/pci/controller/cadence/pcie-cadence-host.c b/drivers/pci/controller/cadence/pcie-cadence-host.c index 5b14f7ee3c79..93d9922730af 100644 --- a/drivers/pci/controller/cadence/pcie-cadence-host.c +++ b/drivers/pci/controller/cadence/pcie-cadence-host.c @@ -497,6 +497,30 @@ static int cdns_pcie_host_init(struct device *dev, return cdns_pcie_host_init_address_translation(rc); } +int cdns_pcie_host_link_setup(struct cdns_pcie_rc *rc) +{ + struct cdns_pcie *pcie = &rc->pcie; + struct device *dev = rc->pcie.dev; + int ret; + + if (rc->quirk_detect_quiet_flag) + cdns_pcie_detect_quiet_min_delay_set(&rc->pcie); + + cdns_pcie_host_enable_ptm_response(pcie); + + ret = cdns_pcie_start_link(pcie); + if (ret) { + dev_err(dev, "Failed to start link\n"); + return ret; + } + + ret = cdns_pcie_host_start_link(rc); + if (ret) + dev_dbg(dev, "PCIe link never came up\n"); + + return 0; +} + int cdns_pcie_host_setup(struct cdns_pcie_rc *rc) { struct device *dev = rc->pcie.dev; @@ -533,20 +557,9 @@ int cdns_pcie_host_setup(struct cdns_pcie_rc *rc) return PTR_ERR(rc->cfg_base); rc->cfg_res = res; - if (rc->quirk_detect_quiet_flag) - cdns_pcie_detect_quiet_min_delay_set(&rc->pcie); - - cdns_pcie_host_enable_ptm_response(pcie); - - ret = cdns_pcie_start_link(pcie); - if (ret) { - dev_err(dev, "Failed to start link\n"); - return ret; - } - - ret = cdns_pcie_host_start_link(rc); + ret = cdns_pcie_host_link_setup(rc); if (ret) - dev_dbg(dev, "PCIe link never came up\n"); + return ret; for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) rc->avail_ib_bar[bar] = true; diff --git a/drivers/pci/controller/cadence/pcie-cadence.h b/drivers/pci/controller/cadence/pcie-cadence.h index 7a66a2f815dc..1d37d5f9f811 100644 --- a/drivers/pci/controller/cadence/pcie-cadence.h +++ b/drivers/pci/controller/cadence/pcie-cadence.h @@ -521,10 +521,16 @@ static inline bool cdns_pcie_link_up(struct cdns_pcie *pcie) } #ifdef CONFIG_PCIE_CADENCE_HOST +int cdns_pcie_host_link_setup(struct cdns_pcie_rc *rc); int cdns_pcie_host_setup(struct cdns_pcie_rc *rc); void __iomem *cdns_pci_map_bus(struct pci_bus *bus, unsigned int devfn, int where); #else +static inline int cdns_pcie_host_link_setup(struct cdns_pcie_rc *rc) +{ + return 0; +} + static inline int cdns_pcie_host_setup(struct cdns_pcie_rc *rc) { return 0; From 063c938928dc80c2bfd66f34df48344db22e009b Mon Sep 17 00:00:00 2001 From: Thomas Richard Date: Wed, 19 Jun 2024 12:15:10 +0200 Subject: [PATCH 082/302] PCI: cadence: Set cdns_pcie_host_init() global MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit During the resume sequence of the host, cdns_pcie_host_init() needs to be called, so set it global. The dev function parameter is removed, as it isn't used. Link: https://lore.kernel.org/linux-pci/20240102-j7200-pcie-s2r-v7-2-a2f9156da6c3@bootlin.com Signed-off-by: Thomas Richard Signed-off-by: Krzysztof Wilczyński Reviewed-by: Siddharth Vadapalli --- drivers/pci/controller/cadence/pcie-cadence-host.c | 5 ++--- drivers/pci/controller/cadence/pcie-cadence.h | 6 ++++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/drivers/pci/controller/cadence/pcie-cadence-host.c b/drivers/pci/controller/cadence/pcie-cadence-host.c index 93d9922730af..8af95e9da7ce 100644 --- a/drivers/pci/controller/cadence/pcie-cadence-host.c +++ b/drivers/pci/controller/cadence/pcie-cadence-host.c @@ -485,8 +485,7 @@ static int cdns_pcie_host_init_address_translation(struct cdns_pcie_rc *rc) return cdns_pcie_host_map_dma_ranges(rc); } -static int cdns_pcie_host_init(struct device *dev, - struct cdns_pcie_rc *rc) +int cdns_pcie_host_init(struct cdns_pcie_rc *rc) { int err; @@ -564,7 +563,7 @@ int cdns_pcie_host_setup(struct cdns_pcie_rc *rc) for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) rc->avail_ib_bar[bar] = true; - ret = cdns_pcie_host_init(dev, rc); + ret = cdns_pcie_host_init(rc); if (ret) return ret; diff --git a/drivers/pci/controller/cadence/pcie-cadence.h b/drivers/pci/controller/cadence/pcie-cadence.h index 1d37d5f9f811..bb215aeebf20 100644 --- a/drivers/pci/controller/cadence/pcie-cadence.h +++ b/drivers/pci/controller/cadence/pcie-cadence.h @@ -522,6 +522,7 @@ static inline bool cdns_pcie_link_up(struct cdns_pcie *pcie) #ifdef CONFIG_PCIE_CADENCE_HOST int cdns_pcie_host_link_setup(struct cdns_pcie_rc *rc); +int cdns_pcie_host_init(struct cdns_pcie_rc *rc); int cdns_pcie_host_setup(struct cdns_pcie_rc *rc); void __iomem *cdns_pci_map_bus(struct pci_bus *bus, unsigned int devfn, int where); @@ -531,6 +532,11 @@ static inline int cdns_pcie_host_link_setup(struct cdns_pcie_rc *rc) return 0; } +static inline int cdns_pcie_host_init(struct cdns_pcie_rc *rc) +{ + return 0; +} + static inline int cdns_pcie_host_setup(struct cdns_pcie_rc *rc) { return 0; From 7d7ce746a9e109ab0aa30ad3c6107e601cf17045 Mon Sep 17 00:00:00 2001 From: Thomas Richard Date: Wed, 19 Jun 2024 12:15:11 +0200 Subject: [PATCH 083/302] PCI: j721e: Use dev_err_probe() in the probe() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use dev_err_probe() instead of dev_err() in the probe() function to simplify the code and standardize the error output. Link: https://lore.kernel.org/linux-pci/20240102-j7200-pcie-s2r-v7-3-a2f9156da6c3@bootlin.com Signed-off-by: Thomas Richard Signed-off-by: Krzysztof Wilczyński Reviewed-by: Francesco Dolcini --- drivers/pci/controller/cadence/pci-j721e.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/drivers/pci/controller/cadence/pci-j721e.c b/drivers/pci/controller/cadence/pci-j721e.c index 7f7732e2dcaa..46d47285ce74 100644 --- a/drivers/pci/controller/cadence/pci-j721e.c +++ b/drivers/pci/controller/cadence/pci-j721e.c @@ -519,20 +519,20 @@ static int j721e_pcie_probe(struct platform_device *pdev) pm_runtime_enable(dev); ret = pm_runtime_get_sync(dev); if (ret < 0) { - dev_err(dev, "pm_runtime_get_sync failed\n"); + dev_err_probe(dev, ret, "pm_runtime_get_sync failed\n"); goto err_get_sync; } ret = j721e_pcie_ctrl_init(pcie); if (ret < 0) { - dev_err(dev, "pm_runtime_get_sync failed\n"); + dev_err_probe(dev, ret, "pm_runtime_get_sync failed\n"); goto err_get_sync; } ret = devm_request_irq(dev, irq, j721e_pcie_link_irq_handler, 0, "j721e-pcie-link-down-irq", pcie); if (ret < 0) { - dev_err(dev, "failed to request link state IRQ %d\n", irq); + dev_err_probe(dev, ret, "failed to request link state IRQ %d\n", irq); goto err_get_sync; } @@ -542,28 +542,25 @@ static int j721e_pcie_probe(struct platform_device *pdev) case PCI_MODE_RC: gpiod = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_LOW); if (IS_ERR(gpiod)) { - ret = PTR_ERR(gpiod); - if (ret != -EPROBE_DEFER) - dev_err(dev, "Failed to get reset GPIO\n"); + ret = dev_err_probe(dev, PTR_ERR(gpiod), "Failed to get reset GPIO\n"); goto err_get_sync; } ret = cdns_pcie_init_phy(dev, cdns_pcie); if (ret) { - dev_err(dev, "Failed to init phy\n"); + dev_err_probe(dev, ret, "Failed to init phy\n"); goto err_get_sync; } clk = devm_clk_get_optional(dev, "pcie_refclk"); if (IS_ERR(clk)) { - ret = PTR_ERR(clk); - dev_err(dev, "failed to get pcie_refclk\n"); + ret = dev_err_probe(dev, PTR_ERR(clk), "failed to get pcie_refclk\n"); goto err_pcie_setup; } ret = clk_prepare_enable(clk); if (ret) { - dev_err(dev, "failed to enable pcie_refclk\n"); + dev_err_probe(dev, ret, "failed to enable pcie_refclk\n"); goto err_pcie_setup; } pcie->refclk = clk; @@ -591,7 +588,7 @@ static int j721e_pcie_probe(struct platform_device *pdev) case PCI_MODE_EP: ret = cdns_pcie_init_phy(dev, cdns_pcie); if (ret) { - dev_err(dev, "Failed to init phy\n"); + dev_err_probe(dev, ret, "Failed to init phy\n"); goto err_get_sync; } From b8600b8791cb2b7c8be894846b1ecddba7291680 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Lebrun?= Date: Wed, 19 Jun 2024 12:15:12 +0200 Subject: [PATCH 084/302] PCI: j721e: Add reset GPIO to struct j721e_pcie MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add reset GPIO to struct j721e_pcie, so it can be used at suspend and resume stages. Link: https://lore.kernel.org/linux-pci/20240102-j7200-pcie-s2r-v7-4-a2f9156da6c3@bootlin.com Signed-off-by: Théo Lebrun Signed-off-by: Thomas Richard Signed-off-by: Krzysztof Wilczyński --- drivers/pci/controller/cadence/pci-j721e.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/pci/controller/cadence/pci-j721e.c b/drivers/pci/controller/cadence/pci-j721e.c index 46d47285ce74..c7f40920e061 100644 --- a/drivers/pci/controller/cadence/pci-j721e.c +++ b/drivers/pci/controller/cadence/pci-j721e.c @@ -53,6 +53,7 @@ struct j721e_pcie { u32 mode; u32 num_lanes; u32 max_lanes; + struct gpio_desc *reset_gpio; void __iomem *user_cfg_base; void __iomem *intd_cfg_base; u32 linkdown_irq_regfield; @@ -545,6 +546,7 @@ static int j721e_pcie_probe(struct platform_device *pdev) ret = dev_err_probe(dev, PTR_ERR(gpiod), "Failed to get reset GPIO\n"); goto err_get_sync; } + pcie->reset_gpio = gpiod; ret = cdns_pcie_init_phy(dev, cdns_pcie); if (ret) { From 97e629c8bdf78066f480f0c44365c6ecb44b7301 Mon Sep 17 00:00:00 2001 From: Yoshihiro Shimoda Date: Thu, 22 Aug 2024 15:44:58 +0900 Subject: [PATCH 085/302] dt-bindings: PCI: rcar-gen4-pci-host: Add R-Car V4M compatible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Document bindings for R-Car V4M (R8A779H0) PCIe host module. Link: https://lore.kernel.org/linux-pci/20240822064459.1133748-2-yoshihiro.shimoda.uh@renesas.com Signed-off-by: Yoshihiro Shimoda Signed-off-by: Krzysztof Wilczyński Signed-off-by: Bjorn Helgaas Reviewed-by: Geert Uytterhoeven Acked-by: Krzysztof Kozlowski --- Documentation/devicetree/bindings/pci/rcar-gen4-pci-host.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/pci/rcar-gen4-pci-host.yaml b/Documentation/devicetree/bindings/pci/rcar-gen4-pci-host.yaml index 955c664f1fbb..bb3f843c59d9 100644 --- a/Documentation/devicetree/bindings/pci/rcar-gen4-pci-host.yaml +++ b/Documentation/devicetree/bindings/pci/rcar-gen4-pci-host.yaml @@ -19,6 +19,7 @@ properties: - enum: - renesas,r8a779f0-pcie # R-Car S4-8 - renesas,r8a779g0-pcie # R-Car V4H + - renesas,r8a779h0-pcie # R-Car V4M - const: renesas,rcar-gen4-pcie # R-Car Gen4 reg: From 05a01639b8f5fa42fb28f58cb1142bf11917d3c2 Mon Sep 17 00:00:00 2001 From: Yoshihiro Shimoda Date: Thu, 22 Aug 2024 15:44:59 +0900 Subject: [PATCH 086/302] dt-bindings: PCI: rcar-gen4-pci-ep: Add R-Car V4M compatible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Document bindings for R-Car V4M (R8A779H0) PCIe endpoint module. Link: https://lore.kernel.org/linux-pci/20240822064459.1133748-3-yoshihiro.shimoda.uh@renesas.com Signed-off-by: Yoshihiro Shimoda Signed-off-by: Krzysztof Wilczyński Signed-off-by: Bjorn Helgaas Reviewed-by: Geert Uytterhoeven Acked-by: Krzysztof Kozlowski --- Documentation/devicetree/bindings/pci/rcar-gen4-pci-ep.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/pci/rcar-gen4-pci-ep.yaml b/Documentation/devicetree/bindings/pci/rcar-gen4-pci-ep.yaml index 91b81ac75592..b23293314a6d 100644 --- a/Documentation/devicetree/bindings/pci/rcar-gen4-pci-ep.yaml +++ b/Documentation/devicetree/bindings/pci/rcar-gen4-pci-ep.yaml @@ -19,6 +19,7 @@ properties: - enum: - renesas,r8a779f0-pcie-ep # R-Car S4-8 - renesas,r8a779g0-pcie-ep # R-Car V4H + - renesas,r8a779h0-pcie-ep # R-Car V4M - const: renesas,rcar-gen4-pcie-ep # R-Car Gen4 reg: From 7cb9b5fa218caa899f4288865c4598357bcce4e9 Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Tue, 3 Sep 2024 15:38:25 -0500 Subject: [PATCH 087/302] PCI: endpoint: Fix enum pci_epc_bar_type kerneldoc e01c9797c0eb ("PCI: endpoint: Clean up hardware description for BARs") added enum pci_epc_bar_type with incomplete kerneldoc. Add the missing piece. Signed-off-by: Bjorn Helgaas --- include/linux/pci-epc.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/linux/pci-epc.h b/include/linux/pci-epc.h index 85bdf2adb760..697cc190747d 100644 --- a/include/linux/pci-epc.h +++ b/include/linux/pci-epc.h @@ -149,6 +149,7 @@ struct pci_epc { }; /** + * enum pci_epc_bar_type - configurability of endpoint BAR * @BAR_PROGRAMMABLE: The BAR mask can be configured by the EPC. * @BAR_FIXED: The BAR mask is fixed by the hardware. * @BAR_RESERVED: The BAR should not be touched by an EPF driver. From dd9d80408b7d6041ebaf2346c66d258bf6adceae Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Tue, 3 Sep 2024 15:16:13 -0500 Subject: [PATCH 088/302] PCI: mediatek: Drop excess mtk_pcie.mem kerneldoc description Struct mtk_pcie.mem was removed by 8a26f861b815 ("PCI: mediatek: Use pci_parse_request_of_pci_ranges()"), but the kerneldoc was left. Remove the extra kerneldoc. Signed-off-by: Bjorn Helgaas --- drivers/pci/controller/pcie-mediatek.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/pci/controller/pcie-mediatek.c b/drivers/pci/controller/pcie-mediatek.c index 7fc0d7709b7f..20c638ca532e 100644 --- a/drivers/pci/controller/pcie-mediatek.c +++ b/drivers/pci/controller/pcie-mediatek.c @@ -211,7 +211,6 @@ struct mtk_pcie_port { * @base: IO mapped register base * @cfg: IO mapped register map for PCIe config * @free_ck: free-run reference clock - * @mem: non-prefetchable memory resource * @ports: pointer to PCIe port information * @soc: pointer to SoC-dependent operations */ From c3d95aa93fd8549588097b0701b3835920fd8533 Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Tue, 3 Sep 2024 15:31:33 -0500 Subject: [PATCH 089/302] PCI: cadence: Drop excess cdns_pcie_rc.dev kerneldoc description Struct cdns_pcie_rc once had a .dev member, but it was removed by bd22885aa188 ("PCI: cadence: Refactor driver to use as a core library"). Drop the extra kerneldoc for it. Signed-off-by: Bjorn Helgaas --- drivers/pci/controller/cadence/pcie-cadence.h | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/pci/controller/cadence/pcie-cadence.h b/drivers/pci/controller/cadence/pcie-cadence.h index 7a66a2f815dc..9e6f602ac79f 100644 --- a/drivers/pci/controller/cadence/pcie-cadence.h +++ b/drivers/pci/controller/cadence/pcie-cadence.h @@ -314,7 +314,6 @@ struct cdns_pcie { /** * struct cdns_pcie_rc - private data for this PCIe Root Complex driver * @pcie: Cadence PCIe controller - * @dev: pointer to PCIe device * @cfg_res: start/end offsets in the physical system memory to map PCI * configuration space accesses * @cfg_base: IO mapped window to access the PCI configuration space of a From 8a4db021b9445dd019f1b4e9466c5812ddf9e63a Mon Sep 17 00:00:00 2001 From: Jim Quinlan Date: Thu, 15 Aug 2024 18:57:14 -0400 Subject: [PATCH 090/302] dt-bindings: PCI: brcm,stb-pcie: Change brcmstb maintainer and cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change maintainer: Nicolas has not been active for a while. It also makes sense for a Broadcom employee to be the maintainer as many of the details are privy to Broadcom. Also, alphabetize the compatible strings. Link: https://lore.kernel.org/linux-pci/20240815225731.40276-2-james.quinlan@broadcom.com Signed-off-by: Jim Quinlan [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Florian Fainelli Reviewed-by: Krzysztof Kozlowski --- Documentation/devicetree/bindings/pci/brcm,stb-pcie.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation/devicetree/bindings/pci/brcm,stb-pcie.yaml b/Documentation/devicetree/bindings/pci/brcm,stb-pcie.yaml index 11f8ea33240c..a95760357335 100644 --- a/Documentation/devicetree/bindings/pci/brcm,stb-pcie.yaml +++ b/Documentation/devicetree/bindings/pci/brcm,stb-pcie.yaml @@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: Brcmstb PCIe Host Controller maintainers: - - Nicolas Saenz Julienne + - Jim Quinlan properties: compatible: @@ -16,11 +16,11 @@ properties: - brcm,bcm2711-pcie # The Raspberry Pi 4 - brcm,bcm4908-pcie - brcm,bcm7211-pcie # Broadcom STB version of RPi4 - - brcm,bcm7278-pcie # Broadcom 7278 Arm - brcm,bcm7216-pcie # Broadcom 7216 Arm - - brcm,bcm7445-pcie # Broadcom 7445 Arm + - brcm,bcm7278-pcie # Broadcom 7278 Arm - brcm,bcm7425-pcie # Broadcom 7425 MIPs - brcm,bcm7435-pcie # Broadcom 7435 MIPs + - brcm,bcm7445-pcie # Broadcom 7445 Arm reg: maxItems: 1 From c64e40caf9eab6bbd3f37d106472723349853b4f Mon Sep 17 00:00:00 2001 From: Jim Quinlan Date: Thu, 15 Aug 2024 18:57:15 -0400 Subject: [PATCH 091/302] dt-bindings: PCI: brcm,stb-pcie: Use maxItems for reset controllers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Provide the maxItem property for the reset controllers and drop their superfluous descriptions. Link: https://lore.kernel.org/linux-pci/20240815225731.40276-3-james.quinlan@broadcom.com Signed-off-by: Jim Quinlan [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Florian Fainelli Reviewed-by: Krzysztof Kozlowski --- .../devicetree/bindings/pci/brcm,stb-pcie.yaml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Documentation/devicetree/bindings/pci/brcm,stb-pcie.yaml b/Documentation/devicetree/bindings/pci/brcm,stb-pcie.yaml index a95760357335..7d2552192153 100644 --- a/Documentation/devicetree/bindings/pci/brcm,stb-pcie.yaml +++ b/Documentation/devicetree/bindings/pci/brcm,stb-pcie.yaml @@ -95,6 +95,12 @@ properties: minItems: 1 maxItems: 3 + resets: + maxItems: 1 + + reset-names: + maxItems: 1 + required: - compatible - reg @@ -118,8 +124,7 @@ allOf: then: properties: resets: - items: - - description: reset controller handling the PERST# signal + maxItems: 1 reset-names: items: @@ -136,8 +141,7 @@ allOf: then: properties: resets: - items: - - description: phandle pointing to the RESCAL reset controller + maxItems: 1 reset-names: items: From 56d020296a122f814b65f3fb5b3f90d7c4533042 Mon Sep 17 00:00:00 2001 From: Jim Quinlan Date: Thu, 15 Aug 2024 18:57:16 -0400 Subject: [PATCH 092/302] dt-bindings: PCI: brcm,stb-pcie: Add 7712 SoC description MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add description for the 7712 SoC, a Broadcom STB sibling chip of the Raspberry Pi 5. The 7712 uses three reset controllers: rescal, for PHY reset calibration; bridge, for the bridge between the PCIe bus and the memory bus; and swinit, which is a "soft" initialization of the PCIe HW. Link: https://lore.kernel.org/linux-pci/20240815225731.40276-4-james.quinlan@broadcom.com Signed-off-by: Jim Quinlan [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Florian Fainelli Reviewed-by: Krzysztof Kozlowski Reviewed-by: Manivannan Sadhasivam --- .../bindings/pci/brcm,stb-pcie.yaml | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/pci/brcm,stb-pcie.yaml b/Documentation/devicetree/bindings/pci/brcm,stb-pcie.yaml index 7d2552192153..0925c520195a 100644 --- a/Documentation/devicetree/bindings/pci/brcm,stb-pcie.yaml +++ b/Documentation/devicetree/bindings/pci/brcm,stb-pcie.yaml @@ -21,6 +21,7 @@ properties: - brcm,bcm7425-pcie # Broadcom 7425 MIPs - brcm,bcm7435-pcie # Broadcom 7435 MIPs - brcm,bcm7445-pcie # Broadcom 7445 Arm + - brcm,bcm7712-pcie # Broadcom STB sibling of Rpi 5 reg: maxItems: 1 @@ -96,10 +97,12 @@ properties: maxItems: 3 resets: - maxItems: 1 + minItems: 1 + maxItems: 3 reset-names: - maxItems: 1 + minItems: 1 + maxItems: 3 required: - compatible @@ -151,6 +154,27 @@ allOf: - resets - reset-names + - if: + properties: + compatible: + contains: + const: brcm,bcm7712-pcie + then: + properties: + resets: + minItems: 3 + maxItems: 3 + + reset-names: + items: + - const: rescal + - const: bridge + - const: swinit + + required: + - resets + - reset-names + unevaluatedProperties: false examples: From 64706ba771f5e8f05b26a1293beed23e83a81b25 Mon Sep 17 00:00:00 2001 From: Jim Quinlan Date: Thu, 15 Aug 2024 18:57:17 -0400 Subject: [PATCH 093/302] PCI: brcmstb: Use common error handling code in brcm_pcie_probe() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactor the error handling in the bottom half of the probe function for readability. The invocation of clk_prepare_enable() is moved lower in the function and this simplifies a couple of return paths. The dev_err_probe() is also used when it is apt. Link: https://lore.kernel.org/linux-pci/20240815225731.40276-5-james.quinlan@broadcom.com Signed-off-by: Jim Quinlan [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Florian Fainelli Reviewed-by: Manivannan Sadhasivam --- drivers/pci/controller/pcie-brcmstb.c | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/controller/pcie-brcmstb.c index c08683febdd4..790a149f6581 100644 --- a/drivers/pci/controller/pcie-brcmstb.c +++ b/drivers/pci/controller/pcie-brcmstb.c @@ -1613,25 +1613,23 @@ static int brcm_pcie_probe(struct platform_device *pdev) pcie->ssc = of_property_read_bool(np, "brcm,enable-ssc"); - ret = clk_prepare_enable(pcie->clk); - if (ret) { - dev_err(&pdev->dev, "could not enable clock\n"); - return ret; - } pcie->rescal = devm_reset_control_get_optional_shared(&pdev->dev, "rescal"); - if (IS_ERR(pcie->rescal)) { - clk_disable_unprepare(pcie->clk); + if (IS_ERR(pcie->rescal)) return PTR_ERR(pcie->rescal); - } + pcie->perst_reset = devm_reset_control_get_optional_exclusive(&pdev->dev, "perst"); - if (IS_ERR(pcie->perst_reset)) { - clk_disable_unprepare(pcie->clk); + if (IS_ERR(pcie->perst_reset)) return PTR_ERR(pcie->perst_reset); - } + + ret = clk_prepare_enable(pcie->clk); + if (ret) + return dev_err_probe(&pdev->dev, ret, "could not enable clock\n"); ret = reset_control_reset(pcie->rescal); - if (ret) - dev_err(&pdev->dev, "failed to deassert 'rescal'\n"); + if (ret) { + clk_disable_unprepare(pcie->clk); + return dev_err_probe(&pdev->dev, ret, "failed to deassert 'rescal'\n"); + } ret = brcm_phy_start(pcie); if (ret) { @@ -1678,6 +1676,7 @@ static int brcm_pcie_probe(struct platform_device *pdev) fail: __brcm_pcie_remove(pcie); + return ret; } From 46cb27f671f2148005ee556b895d6372c6608e27 Mon Sep 17 00:00:00 2001 From: Jim Quinlan Date: Thu, 15 Aug 2024 18:57:18 -0400 Subject: [PATCH 094/302] PCI: brcmstb: Use bridge reset if available MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 7712 SoC has a bridge reset which can be described in the device tree. Use it if present. Otherwise, continue to use the legacy method to reset the bridge. Link: https://lore.kernel.org/linux-pci/20240815225731.40276-6-james.quinlan@broadcom.com Signed-off-by: Jim Quinlan [kwilczynski: commit log, refactored function brcm_pcie_bridge_sw_init_set_generic()] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Florian Fainelli Reviewed-by: Manivannan Sadhasivam Reviewed-by: Stanimir Varbanov --- drivers/pci/controller/pcie-brcmstb.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/controller/pcie-brcmstb.c index 790a149f6581..f8b6765d2e65 100644 --- a/drivers/pci/controller/pcie-brcmstb.c +++ b/drivers/pci/controller/pcie-brcmstb.c @@ -265,6 +265,7 @@ struct brcm_pcie { enum pcie_type type; struct reset_control *rescal; struct reset_control *perst_reset; + struct reset_control *bridge_reset; int num_memc; u64 memc_size[PCIE_BRCM_MAX_MEMC]; u32 hw_rev; @@ -732,9 +733,18 @@ static void __iomem *brcm7425_pcie_map_bus(struct pci_bus *bus, static void brcm_pcie_bridge_sw_init_set_generic(struct brcm_pcie *pcie, u32 val) { - u32 tmp, mask = RGR1_SW_INIT_1_INIT_GENERIC_MASK; + u32 tmp, mask = RGR1_SW_INIT_1_INIT_GENERIC_MASK; u32 shift = RGR1_SW_INIT_1_INIT_GENERIC_SHIFT; + if (pcie->bridge_reset) { + if (val) + reset_control_assert(pcie->bridge_reset); + else + reset_control_deassert(pcie->bridge_reset); + + return; + } + tmp = readl(pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); tmp = (tmp & ~mask) | ((val << shift) & mask); writel(tmp, pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); @@ -1621,10 +1631,16 @@ static int brcm_pcie_probe(struct platform_device *pdev) if (IS_ERR(pcie->perst_reset)) return PTR_ERR(pcie->perst_reset); + pcie->bridge_reset = devm_reset_control_get_optional_exclusive(&pdev->dev, "bridge"); + if (IS_ERR(pcie->bridge_reset)) + return PTR_ERR(pcie->bridge_reset); + ret = clk_prepare_enable(pcie->clk); if (ret) return dev_err_probe(&pdev->dev, ret, "could not enable clock\n"); + pcie->bridge_sw_init_set(pcie, 0); + ret = reset_control_reset(pcie->rescal); if (ret) { clk_disable_unprepare(pcie->clk); From 8201360218c6a42b3f7ff03d8908bd345e3620f4 Mon Sep 17 00:00:00 2001 From: Jim Quinlan Date: Thu, 15 Aug 2024 18:57:19 -0400 Subject: [PATCH 095/302] PCI: brcmstb: Use swinit reset if available MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 7712 SoC adds a software init reset device for the PCIe HW. If found in the DT node, use it. Link: https://lore.kernel.org/linux-pci/20240815225731.40276-7-james.quinlan@broadcom.com Signed-off-by: Jim Quinlan Signed-off-by: Krzysztof Wilczyński Reviewed-by: Florian Fainelli Reviewed-by: Manivannan Sadhasivam --- drivers/pci/controller/pcie-brcmstb.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/controller/pcie-brcmstb.c index f8b6765d2e65..926644eff39d 100644 --- a/drivers/pci/controller/pcie-brcmstb.c +++ b/drivers/pci/controller/pcie-brcmstb.c @@ -266,6 +266,7 @@ struct brcm_pcie { struct reset_control *rescal; struct reset_control *perst_reset; struct reset_control *bridge_reset; + struct reset_control *swinit_reset; int num_memc; u64 memc_size[PCIE_BRCM_MAX_MEMC]; u32 hw_rev; @@ -1635,12 +1636,35 @@ static int brcm_pcie_probe(struct platform_device *pdev) if (IS_ERR(pcie->bridge_reset)) return PTR_ERR(pcie->bridge_reset); + pcie->swinit_reset = devm_reset_control_get_optional_exclusive(&pdev->dev, "swinit"); + if (IS_ERR(pcie->swinit_reset)) + return PTR_ERR(pcie->swinit_reset); + ret = clk_prepare_enable(pcie->clk); if (ret) return dev_err_probe(&pdev->dev, ret, "could not enable clock\n"); pcie->bridge_sw_init_set(pcie, 0); + if (pcie->swinit_reset) { + ret = reset_control_assert(pcie->swinit_reset); + if (ret) { + clk_disable_unprepare(pcie->clk); + return dev_err_probe(&pdev->dev, ret, + "could not assert reset 'swinit'\n"); + } + + /* HW team recommends 1us for proper sync and propagation of reset */ + udelay(1); + + ret = reset_control_deassert(pcie->swinit_reset); + if (ret) { + clk_disable_unprepare(pcie->clk); + return dev_err_probe(&pdev->dev, ret, + "could not de-assert reset 'swinit'\n"); + } + } + ret = reset_control_reset(pcie->rescal); if (ret) { clk_disable_unprepare(pcie->clk); From 30eb2080fe2043c3e61c1ae4bb6917800b10fb08 Mon Sep 17 00:00:00 2001 From: Jim Quinlan Date: Thu, 15 Aug 2024 18:57:20 -0400 Subject: [PATCH 096/302] PCI: brcmstb: PCI: brcmstb: Make HARD_DEBUG, INTR2_CPU_BASE offsets SoC-specific MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Do preparatory work for the 7712 SoC, which is introduced in a future commit. Our HW design has changed two register offsets for the 7712, where previously it was a common value for all Broadcom SoCs with PCIe cores. Specifically, the two offsets are to the registers HARD_DEBUG and INTR2_CPU_BASE. Link: https://lore.kernel.org/linux-pci/20240815225731.40276-8-james.quinlan@broadcom.com Signed-off-by: Jim Quinlan [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński Tested-by: Florian Fainelli Reviewed-by: Florian Fainelli Reviewed-by: Krzysztof Kozlowski Reviewed-by: Manivannan Sadhasivam Reviewed-by: Stanimir Varbanov --- drivers/pci/controller/pcie-brcmstb.c | 39 ++++++++++++++++----------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/controller/pcie-brcmstb.c index 926644eff39d..9d31dd29e831 100644 --- a/drivers/pci/controller/pcie-brcmstb.c +++ b/drivers/pci/controller/pcie-brcmstb.c @@ -122,7 +122,6 @@ #define PCIE_MEM_WIN0_LIMIT_HI(win) \ PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI + ((win) * 8) -#define PCIE_MISC_HARD_PCIE_HARD_DEBUG 0x4204 #define PCIE_MISC_HARD_PCIE_HARD_DEBUG_CLKREQ_DEBUG_ENABLE_MASK 0x2 #define PCIE_MISC_HARD_PCIE_HARD_DEBUG_L1SS_ENABLE_MASK 0x200000 #define PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK 0x08000000 @@ -131,9 +130,9 @@ (PCIE_MISC_HARD_PCIE_HARD_DEBUG_CLKREQ_DEBUG_ENABLE_MASK | \ PCIE_MISC_HARD_PCIE_HARD_DEBUG_L1SS_ENABLE_MASK) -#define PCIE_INTR2_CPU_BASE 0x4300 #define PCIE_MSI_INTR2_BASE 0x4500 -/* Offsets from PCIE_INTR2_CPU_BASE and PCIE_MSI_INTR2_BASE */ + +/* Offsets from INTR2_CPU and MSI_INTR2 BASE offsets */ #define MSI_INT_STATUS 0x0 #define MSI_INT_CLR 0x8 #define MSI_INT_MASK_SET 0x10 @@ -184,9 +183,11 @@ #define SSC_STATUS_PLL_LOCK_MASK 0x800 #define PCIE_BRCM_MAX_MEMC 3 -#define IDX_ADDR(pcie) (pcie->reg_offsets[EXT_CFG_INDEX]) -#define DATA_ADDR(pcie) (pcie->reg_offsets[EXT_CFG_DATA]) -#define PCIE_RGR1_SW_INIT_1(pcie) (pcie->reg_offsets[RGR1_SW_INIT_1]) +#define IDX_ADDR(pcie) ((pcie)->reg_offsets[EXT_CFG_INDEX]) +#define DATA_ADDR(pcie) ((pcie)->reg_offsets[EXT_CFG_DATA]) +#define PCIE_RGR1_SW_INIT_1(pcie) ((pcie)->reg_offsets[RGR1_SW_INIT_1]) +#define HARD_DEBUG(pcie) ((pcie)->reg_offsets[PCIE_HARD_DEBUG]) +#define INTR2_CPU_BASE(pcie) ((pcie)->reg_offsets[PCIE_INTR2_CPU_BASE]) /* Rescal registers */ #define PCIE_DVT_PMU_PCIE_PHY_CTRL 0xc700 @@ -205,6 +206,8 @@ enum { RGR1_SW_INIT_1, EXT_CFG_INDEX, EXT_CFG_DATA, + PCIE_HARD_DEBUG, + PCIE_INTR2_CPU_BASE, }; enum { @@ -651,7 +654,7 @@ static int brcm_pcie_enable_msi(struct brcm_pcie *pcie) BUILD_BUG_ON(BRCM_INT_PCI_MSI_LEGACY_NR > BRCM_INT_PCI_MSI_NR); if (msi->legacy) { - msi->intr_base = msi->base + PCIE_INTR2_CPU_BASE; + msi->intr_base = msi->base + INTR2_CPU_BASE(pcie); msi->nr = BRCM_INT_PCI_MSI_LEGACY_NR; msi->legacy_shift = 24; } else { @@ -900,12 +903,12 @@ static int brcm_pcie_setup(struct brcm_pcie *pcie) /* Take the bridge out of reset */ pcie->bridge_sw_init_set(pcie, 0); - tmp = readl(base + PCIE_MISC_HARD_PCIE_HARD_DEBUG); + tmp = readl(base + HARD_DEBUG(pcie)); if (is_bmips(pcie)) tmp &= ~PCIE_BMIPS_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK; else tmp &= ~PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK; - writel(tmp, base + PCIE_MISC_HARD_PCIE_HARD_DEBUG); + writel(tmp, base + HARD_DEBUG(pcie)); /* Wait for SerDes to be stable */ usleep_range(100, 200); @@ -1074,7 +1077,7 @@ static void brcm_config_clkreq(struct brcm_pcie *pcie) } /* Start out assuming safe mode (both mode bits cleared) */ - clkreq_cntl = readl(pcie->base + PCIE_MISC_HARD_PCIE_HARD_DEBUG); + clkreq_cntl = readl(pcie->base + HARD_DEBUG(pcie)); clkreq_cntl &= ~PCIE_CLKREQ_MASK; if (strcmp(mode, "no-l1ss") == 0) { @@ -1117,7 +1120,7 @@ static void brcm_config_clkreq(struct brcm_pcie *pcie) dev_err(pcie->dev, err_msg); mode = "safe"; } - writel(clkreq_cntl, pcie->base + PCIE_MISC_HARD_PCIE_HARD_DEBUG); + writel(clkreq_cntl, pcie->base + HARD_DEBUG(pcie)); dev_info(pcie->dev, "clkreq-mode set to %s\n", mode); } @@ -1339,9 +1342,9 @@ static void brcm_pcie_turn_off(struct brcm_pcie *pcie) writel(tmp, base + PCIE_MISC_PCIE_CTRL); /* Turn off SerDes */ - tmp = readl(base + PCIE_MISC_HARD_PCIE_HARD_DEBUG); + tmp = readl(base + HARD_DEBUG(pcie)); u32p_replace_bits(&tmp, 1, PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK); - writel(tmp, base + PCIE_MISC_HARD_PCIE_HARD_DEBUG); + writel(tmp, base + HARD_DEBUG(pcie)); /* Shutdown PCIe bridge */ pcie->bridge_sw_init_set(pcie, 1); @@ -1427,9 +1430,9 @@ static int brcm_pcie_resume_noirq(struct device *dev) pcie->bridge_sw_init_set(pcie, 0); /* SERDES_IDDQ = 0 */ - tmp = readl(base + PCIE_MISC_HARD_PCIE_HARD_DEBUG); + tmp = readl(base + HARD_DEBUG(pcie)); u32p_replace_bits(&tmp, 0, PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK); - writel(tmp, base + PCIE_MISC_HARD_PCIE_HARD_DEBUG); + writel(tmp, base + HARD_DEBUG(pcie)); /* wait for serdes to be stable */ udelay(100); @@ -1501,12 +1504,16 @@ static const int pcie_offsets[] = { [RGR1_SW_INIT_1] = 0x9210, [EXT_CFG_INDEX] = 0x9000, [EXT_CFG_DATA] = 0x9004, + [PCIE_HARD_DEBUG] = 0x4204, + [PCIE_INTR2_CPU_BASE] = 0x4300, }; static const int pcie_offsets_bmips_7425[] = { [RGR1_SW_INIT_1] = 0x8010, [EXT_CFG_INDEX] = 0x8300, [EXT_CFG_DATA] = 0x8304, + [PCIE_HARD_DEBUG] = 0x4204, + [PCIE_INTR2_CPU_BASE] = 0x4300, }; static const struct pcie_cfg_data generic_cfg = { @@ -1541,6 +1548,8 @@ static const int pcie_offset_bcm7278[] = { [RGR1_SW_INIT_1] = 0xc010, [EXT_CFG_INDEX] = 0x9000, [EXT_CFG_DATA] = 0x9004, + [PCIE_HARD_DEBUG] = 0x4204, + [PCIE_INTR2_CPU_BASE] = 0x4300, }; static const struct pcie_cfg_data bcm7278_cfg = { From 0d8046037610dcf9e59ece73bf4bbcec1e6a878b Mon Sep 17 00:00:00 2001 From: Jim Quinlan Date: Thu, 15 Aug 2024 18:57:21 -0400 Subject: [PATCH 097/302] PCI: brcmstb: Remove two unused constants from driver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove two constants in the driver which are no longer used: RGR1_SW_INIT_1_INIT_MASK and RGR1_SW_INIT_1_INIT_SHIFT. Link: https://lore.kernel.org/linux-pci/20240815225731.40276-9-james.quinlan@broadcom.com Signed-off-by: Jim Quinlan Signed-off-by: Krzysztof Wilczyński Tested-by: Florian Fainelli Reviewed-by: Stanimir Varbanov Reviewed-by: Florian Fainelli Reviewed-by: Manivannan Sadhasivam Reviewed-by: Krzysztof Kozlowski --- drivers/pci/controller/pcie-brcmstb.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/controller/pcie-brcmstb.c index 9d31dd29e831..510d982bbf92 100644 --- a/drivers/pci/controller/pcie-brcmstb.c +++ b/drivers/pci/controller/pcie-brcmstb.c @@ -210,11 +210,6 @@ enum { PCIE_INTR2_CPU_BASE, }; -enum { - RGR1_SW_INIT_1_INIT_MASK, - RGR1_SW_INIT_1_INIT_SHIFT, -}; - enum pcie_type { GENERIC, BCM7425, From e1c88956e200e225f2712de4b5e2be923cf559fc Mon Sep 17 00:00:00 2001 From: Jim Quinlan Date: Thu, 15 Aug 2024 18:57:22 -0400 Subject: [PATCH 098/302] PCI: brcmstb: Don't conflate the reset rescal with PHY ctrl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a "has_phy" field indicating that the internal PHY has SW control that requires configuration. Some previous chips only required the firing of the "rescal" reset controller. This change requires us to give the 7216 SoC its own cfg_data structure. Link: https://lore.kernel.org/linux-pci/20240815225731.40276-10-james.quinlan@broadcom.com Signed-off-by: Jim Quinlan [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński Tested-by: Florian Fainelli Reviewed-by: Florian Fainelli Reviewed-by: Krzysztof Kozlowski Reviewed-by: Manivannan Sadhasivam Reviewed-by: Stanimir Varbanov --- drivers/pci/controller/pcie-brcmstb.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/controller/pcie-brcmstb.c index 510d982bbf92..f1d890aea369 100644 --- a/drivers/pci/controller/pcie-brcmstb.c +++ b/drivers/pci/controller/pcie-brcmstb.c @@ -222,6 +222,7 @@ enum pcie_type { struct pcie_cfg_data { const int *offsets; const enum pcie_type type; + const bool has_phy; void (*perst_set)(struct brcm_pcie *pcie, u32 val); void (*bridge_sw_init_set)(struct brcm_pcie *pcie, u32 val); }; @@ -272,6 +273,7 @@ struct brcm_pcie { void (*bridge_sw_init_set)(struct brcm_pcie *pcie, u32 val); struct subdev_regulators *sr; bool ep_wakeup_capable; + bool has_phy; }; static inline bool is_bmips(const struct brcm_pcie *pcie) @@ -1313,12 +1315,12 @@ static int brcm_phy_cntl(struct brcm_pcie *pcie, const int start) static inline int brcm_phy_start(struct brcm_pcie *pcie) { - return pcie->rescal ? brcm_phy_cntl(pcie, 1) : 0; + return pcie->has_phy ? brcm_phy_cntl(pcie, 1) : 0; } static inline int brcm_phy_stop(struct brcm_pcie *pcie) { - return pcie->rescal ? brcm_phy_cntl(pcie, 0) : 0; + return pcie->has_phy ? brcm_phy_cntl(pcie, 0) : 0; } static void brcm_pcie_turn_off(struct brcm_pcie *pcie) @@ -1561,12 +1563,20 @@ static const struct pcie_cfg_data bcm2711_cfg = { .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, }; +static const struct pcie_cfg_data bcm7216_cfg = { + .offsets = pcie_offset_bcm7278, + .type = BCM7278, + .perst_set = brcm_pcie_perst_set_7278, + .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_7278, + .has_phy = true, +}; + static const struct of_device_id brcm_pcie_match[] = { { .compatible = "brcm,bcm2711-pcie", .data = &bcm2711_cfg }, { .compatible = "brcm,bcm4908-pcie", .data = &bcm4908_cfg }, { .compatible = "brcm,bcm7211-pcie", .data = &generic_cfg }, { .compatible = "brcm,bcm7278-pcie", .data = &bcm7278_cfg }, - { .compatible = "brcm,bcm7216-pcie", .data = &bcm7278_cfg }, + { .compatible = "brcm,bcm7216-pcie", .data = &bcm7216_cfg }, { .compatible = "brcm,bcm7445-pcie", .data = &generic_cfg }, { .compatible = "brcm,bcm7435-pcie", .data = &bcm7435_cfg }, { .compatible = "brcm,bcm7425-pcie", .data = &bcm7425_cfg }, @@ -1614,6 +1624,7 @@ static int brcm_pcie_probe(struct platform_device *pdev) pcie->type = data->type; pcie->perst_set = data->perst_set; pcie->bridge_sw_init_set = data->bridge_sw_init_set; + pcie->has_phy = data->has_phy; pcie->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pcie->base)) From 6aa9c09f1bcd339749b249830c604871740df268 Mon Sep 17 00:00:00 2001 From: Thomas Richard Date: Wed, 19 Jun 2024 12:15:13 +0200 Subject: [PATCH 099/302] PCI: Add T_PERST_CLK_US macro MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "Power Sequencing and Reset Signal Timings" table of the PCI Express Card Electromechanical Specification, Revision 5.1, Section 2.9.2, indicates PERST# should be deasserted after minimum of 100us once REFCLK is stable (symbol T_PERST-CLK). Add a macro so that PCIe controller drivers can use it. Link: https://lore.kernel.org/linux-pci/20240102-j7200-pcie-s2r-v7-5-a2f9156da6c3@bootlin.com Signed-off-by: Thomas Richard [kwilczynski: commit log, update sleep interval macros code comments] Signed-off-by: Krzysztof Wilczyński --- drivers/pci/pci.h | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 79c8398f3938..abfe0b3d5f1e 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -13,9 +13,24 @@ #define PCIE_LINK_RETRAIN_TIMEOUT_MS 1000 -/* Power stable to PERST# inactive from PCIe card Electromechanical Spec */ +/* + * Power stable to PERST# inactive. + * + * See the "Power Sequencing and Reset Signal Timings" table of the PCI Express + * Card Electromechanical Specification, Revision 5.1, Section 2.9.2, Symbol + * "T_PVPERL". + */ #define PCIE_T_PVPERL_MS 100 +/* + * REFCLK stable before PERST# inactive. + * + * See the "Power Sequencing and Reset Signal Timings" table of the PCI Express + * Card Electromechanical Specification, Revision 5.1, Section 2.9.2, Symbol + * "T_PERST-CLK". + */ +#define PCIE_T_PERST_CLK_US 100 + /* * End of conventional reset (PERST# de-asserted) to first configuration * request (device able to respond with a "Request Retry Status" completion), From f96b6971373382855bc964f1c067bd6dc41cf0ab Mon Sep 17 00:00:00 2001 From: Thomas Richard Date: Wed, 19 Jun 2024 12:15:14 +0200 Subject: [PATCH 100/302] PCI: j721e: Use T_PERST_CLK_US macro MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the T_PERST_CLK_US macro, and the fsleep() function instead of usleep_range(). Link: https://lore.kernel.org/linux-pci/20240102-j7200-pcie-s2r-v7-6-a2f9156da6c3@bootlin.com Signed-off-by: Thomas Richard Signed-off-by: Krzysztof Wilczyński --- drivers/pci/controller/cadence/pci-j721e.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/pci/controller/cadence/pci-j721e.c b/drivers/pci/controller/cadence/pci-j721e.c index c7f40920e061..4096cdd6e8ee 100644 --- a/drivers/pci/controller/cadence/pci-j721e.c +++ b/drivers/pci/controller/cadence/pci-j721e.c @@ -576,7 +576,7 @@ static int j721e_pcie_probe(struct platform_device *pdev) * after 100 us. */ if (gpiod) { - usleep_range(100, 200); + fsleep(PCIE_T_PERST_CLK_US); gpiod_set_value_cansleep(gpiod, 1); } From c538d40f365b5b6d7433d371710f58e8b266fb19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Lebrun?= Date: Wed, 19 Jun 2024 12:15:15 +0200 Subject: [PATCH 101/302] PCI: j721e: Add suspend and resume support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add suspend and resume support. Only the Root Complex mode is supported. During the suspend stage PERST# is asserted, then deasserted during the resume stage. Link: https://lore.kernel.org/linux-pci/20240102-j7200-pcie-s2r-v7-7-a2f9156da6c3@bootlin.com Signed-off-by: Théo Lebrun Signed-off-by: Thomas Richard [kwilczynski: commit log, update references to the PCI SIG specification] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Siddharth Vadapalli --- drivers/pci/controller/cadence/pci-j721e.c | 98 ++++++++++++++++++++-- 1 file changed, 92 insertions(+), 6 deletions(-) diff --git a/drivers/pci/controller/cadence/pci-j721e.c b/drivers/pci/controller/cadence/pci-j721e.c index 4096cdd6e8ee..284f2e0e4d26 100644 --- a/drivers/pci/controller/cadence/pci-j721e.c +++ b/drivers/pci/controller/cadence/pci-j721e.c @@ -7,6 +7,8 @@ */ #include +#include +#include #include #include #include @@ -22,6 +24,8 @@ #include "../../pci.h" #include "pcie-cadence.h" +#define cdns_pcie_to_rc(p) container_of(p, struct cdns_pcie_rc, pcie) + #define ENABLE_REG_SYS_2 0x108 #define STATUS_REG_SYS_2 0x508 #define STATUS_CLR_REG_SYS_2 0x708 @@ -568,12 +572,12 @@ static int j721e_pcie_probe(struct platform_device *pdev) pcie->refclk = clk; /* - * "Power Sequencing and Reset Signal Timings" table in - * PCI EXPRESS CARD ELECTROMECHANICAL SPECIFICATION, REV. 3.0 - * indicates PERST# should be deasserted after minimum of 100us - * once REFCLK is stable. The REFCLK to the connector in RC - * mode is selected while enabling the PHY. So deassert PERST# - * after 100 us. + * The "Power Sequencing and Reset Signal Timings" table of the + * PCI Express Card Electromechanical Specification, Revision + * 5.1, Section 2.9.2, Symbol "T_PERST-CLK", indicates PERST# + * should be deasserted after minimum of 100us once REFCLK is + * stable. The REFCLK to the connector in RC mode is selected + * while enabling the PHY. So deassert PERST# after 100 us. */ if (gpiod) { fsleep(PCIE_T_PERST_CLK_US); @@ -625,6 +629,87 @@ static void j721e_pcie_remove(struct platform_device *pdev) pm_runtime_disable(dev); } +static int j721e_pcie_suspend_noirq(struct device *dev) +{ + struct j721e_pcie *pcie = dev_get_drvdata(dev); + + if (pcie->mode == PCI_MODE_RC) { + gpiod_set_value_cansleep(pcie->reset_gpio, 0); + clk_disable_unprepare(pcie->refclk); + } + + cdns_pcie_disable_phy(pcie->cdns_pcie); + + return 0; +} + +static int j721e_pcie_resume_noirq(struct device *dev) +{ + struct j721e_pcie *pcie = dev_get_drvdata(dev); + struct cdns_pcie *cdns_pcie = pcie->cdns_pcie; + int ret; + + ret = j721e_pcie_ctrl_init(pcie); + if (ret < 0) + return ret; + + j721e_pcie_config_link_irq(pcie); + + /* + * This is not called explicitly in the probe, it is called by + * cdns_pcie_init_phy(). + */ + ret = cdns_pcie_enable_phy(pcie->cdns_pcie); + if (ret < 0) + return ret; + + if (pcie->mode == PCI_MODE_RC) { + struct cdns_pcie_rc *rc = cdns_pcie_to_rc(cdns_pcie); + + ret = clk_prepare_enable(pcie->refclk); + if (ret < 0) + return ret; + + /* + * The "Power Sequencing and Reset Signal Timings" table of the + * PCI Express Card Electromechanical Specification, Revision + * 5.1, Section 2.9.2, Symbol "T_PERST-CLK", indicates PERST# + * should be deasserted after minimum of 100us once REFCLK is + * stable. The REFCLK to the connector in RC mode is selected + * while enabling the PHY. So deassert PERST# after 100 us. + */ + if (pcie->reset_gpio) { + fsleep(PCIE_T_PERST_CLK_US); + gpiod_set_value_cansleep(pcie->reset_gpio, 1); + } + + ret = cdns_pcie_host_link_setup(rc); + if (ret < 0) { + clk_disable_unprepare(pcie->refclk); + return ret; + } + + /* + * Reset internal status of BARs to force reinitialization in + * cdns_pcie_host_init(). + */ + for (enum cdns_pcie_rp_bar bar = RP_BAR0; bar <= RP_NO_BAR; bar++) + rc->avail_ib_bar[bar] = true; + + ret = cdns_pcie_host_init(rc); + if (ret) { + clk_disable_unprepare(pcie->refclk); + return ret; + } + } + + return 0; +} + +static DEFINE_NOIRQ_DEV_PM_OPS(j721e_pcie_pm_ops, + j721e_pcie_suspend_noirq, + j721e_pcie_resume_noirq); + static struct platform_driver j721e_pcie_driver = { .probe = j721e_pcie_probe, .remove_new = j721e_pcie_remove, @@ -632,6 +717,7 @@ static struct platform_driver j721e_pcie_driver = { .name = "j721e-pcie", .of_match_table = of_j721e_pcie_match, .suppress_bind_attrs = true, + .pm = pm_sleep_ptr(&j721e_pcie_pm_ops), }, }; builtin_platform_driver(j721e_pcie_driver); From ac44be2155cd1e307a3c18af62745e6736817369 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Sun, 18 Aug 2024 19:28:41 +0200 Subject: [PATCH 102/302] dt-bindings: PCI: hisilicon,kirin-pcie: Add top-level constraints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Properties with variable number of items per each device are expected to have widest constraints in top-level "properties:" block and further customized (narrowed) in "if:then:". Add missing top-level constraints for clock-names and reset-names. Link: https://lore.kernel.org/linux-pci/20240818172843.121787-1-krzysztof.kozlowski@linaro.org Signed-off-by: Krzysztof Kozlowski Signed-off-by: Krzysztof Wilczyński Reviewed-by: Rob Herring (Arm) --- .../devicetree/bindings/pci/hisilicon,kirin-pcie.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/pci/hisilicon,kirin-pcie.yaml b/Documentation/devicetree/bindings/pci/hisilicon,kirin-pcie.yaml index c9f04999c9cf..e863519f3161 100644 --- a/Documentation/devicetree/bindings/pci/hisilicon,kirin-pcie.yaml +++ b/Documentation/devicetree/bindings/pci/hisilicon,kirin-pcie.yaml @@ -37,7 +37,8 @@ properties: minItems: 3 maxItems: 4 - clocks: true + clocks: + maxItems: 5 clock-names: items: From c62a0b8fe8bfbaa78efe5de7b30a9d0d225be1ab Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Sun, 18 Aug 2024 19:28:42 +0200 Subject: [PATCH 103/302] dt-bindings: PCI: renesas,pci-rcar-gen2: Add top-level constraints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Properties with variable number of items per each device are expected to have widest constraints in top-level "properties:" block and further customized (narrowed) in "if:then:". Add missing top-level constraints for clocks and clock-names. Link: https://lore.kernel.org/linux-pci/20240818172843.121787-2-krzysztof.kozlowski@linaro.org Signed-off-by: Krzysztof Kozlowski Signed-off-by: Krzysztof Wilczyński Reviewed-by: Rob Herring (Arm) --- .../devicetree/bindings/pci/renesas,pci-rcar-gen2.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/pci/renesas,pci-rcar-gen2.yaml b/Documentation/devicetree/bindings/pci/renesas,pci-rcar-gen2.yaml index b288cdb1ec70..065b7508d288 100644 --- a/Documentation/devicetree/bindings/pci/renesas,pci-rcar-gen2.yaml +++ b/Documentation/devicetree/bindings/pci/renesas,pci-rcar-gen2.yaml @@ -42,9 +42,13 @@ properties: interrupts: maxItems: 1 - clocks: true + clocks: + minItems: 1 + maxItems: 3 - clock-names: true + clock-names: + minItems: 1 + maxItems: 3 resets: maxItems: 1 From a5c1bf7e9a4638fbb27461e9801f07204b50dcb6 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Sun, 18 Aug 2024 19:28:43 +0200 Subject: [PATCH 104/302] dt-bindings: PCI: socionext,uniphier-pcie-ep: Add top-level constraints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Properties with variable number of items per each device are expected to have widest constraints in top-level "properties:" block and further customized (narrowed) in "if:then:". Add missing top-level constraints for clock-names and reset-names. Link: https://lore.kernel.org/linux-pci/20240818172843.121787-3-krzysztof.kozlowski@linaro.org Signed-off-by: Krzysztof Kozlowski Signed-off-by: Krzysztof Wilczyński Reviewed-by: Kunihiko Hayashi Reviewed-by: Rob Herring (Arm) --- .../bindings/pci/socionext,uniphier-pcie-ep.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/pci/socionext,uniphier-pcie-ep.yaml b/Documentation/devicetree/bindings/pci/socionext,uniphier-pcie-ep.yaml index f0d8e486a07d..93f3d0f4bb94 100644 --- a/Documentation/devicetree/bindings/pci/socionext,uniphier-pcie-ep.yaml +++ b/Documentation/devicetree/bindings/pci/socionext,uniphier-pcie-ep.yaml @@ -38,13 +38,17 @@ properties: minItems: 1 maxItems: 2 - clock-names: true + clock-names: + minItems: 1 + maxItems: 2 resets: minItems: 1 maxItems: 2 - reset-names: true + reset-names: + minItems: 1 + maxItems: 2 num-ib-windows: const: 16 From 1a1bf58897d20fc38b454dfecd2771e142d36966 Mon Sep 17 00:00:00 2001 From: Frank Li Date: Mon, 26 Aug 2024 17:38:32 -0400 Subject: [PATCH 105/302] dt-bindings: PCI: layerscape-pci: Replace fsl,lx2160a-pcie with fsl,lx2160ar2-pcie MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fsl,lx2160a-pcie compatible is used for mobivel according to the Documentation/devicetree/bindings/pci/layerscape-pcie-gen4.txt file. Whereas the fsl,layerscape-pcie is used for DesignWare PCIe controller binding. So change it to fsl,lx2160ar2-pcie and allow a fall back to fsl,ls2088a-pcie. While at it, sort compatible string. Fixes: 24cd7ecb3886 ("dt-bindings: PCI: layerscape-pci: Convert to YAML format") Link: https://lore.kernel.org/linux-pci/20240826-2160r2-v1-1-106340d538d6@nxp.com Signed-off-by: Frank Li [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Rob Herring (Arm) --- .../bindings/pci/fsl,layerscape-pcie.yaml | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/Documentation/devicetree/bindings/pci/fsl,layerscape-pcie.yaml b/Documentation/devicetree/bindings/pci/fsl,layerscape-pcie.yaml index 793986c5af7f..daeab5c0758d 100644 --- a/Documentation/devicetree/bindings/pci/fsl,layerscape-pcie.yaml +++ b/Documentation/devicetree/bindings/pci/fsl,layerscape-pcie.yaml @@ -22,18 +22,20 @@ description: properties: compatible: - enum: - - fsl,ls1021a-pcie - - fsl,ls2080a-pcie - - fsl,ls2085a-pcie - - fsl,ls2088a-pcie - - fsl,ls1088a-pcie - - fsl,ls1046a-pcie - - fsl,ls1043a-pcie - - fsl,ls1012a-pcie - - fsl,ls1028a-pcie - - fsl,lx2160a-pcie - + oneOf: + - enum: + - fsl,ls1012a-pcie + - fsl,ls1021a-pcie + - fsl,ls1028a-pcie + - fsl,ls1043a-pcie + - fsl,ls1046a-pcie + - fsl,ls1088a-pcie + - fsl,ls2080a-pcie + - fsl,ls2085a-pcie + - fsl,ls2088a-pcie + - items: + - const: fsl,lx2160ar2-pcie + - const: fsl,ls2088a-pcie reg: maxItems: 2 From b9fe09a1b293429f456115dae8cf9cecb82b5d13 Mon Sep 17 00:00:00 2001 From: Frank Li Date: Fri, 23 Aug 2024 14:58:54 -0400 Subject: [PATCH 106/302] dt-bindings: PCI: layerscape-pci: Add deprecated property 'num-viewport' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copy the 'num-viewport' property from snps,dw-pcie-common.yaml to fsl,layerscape-pcie.yaml to address the following warning: /arch/arm64/boot/dts/freescale/fsl-ls1012a-frwy.dtb: pcie@3400000: Unevaluated properties are not allowed ('num-viewport' was unexpected) This is necessary due to historical reasons where fsl,layerscape-pcie.yaml does not directly reference snps,dw-pcie-common.yaml. Link: https://lore.kernel.org/linux-pci/20240823185855.776904-1-Frank.Li@nxp.com Signed-off-by: Frank Li [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński Acked-by: Rob Herring (Arm) --- .../devicetree/bindings/pci/fsl,layerscape-pcie.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Documentation/devicetree/bindings/pci/fsl,layerscape-pcie.yaml b/Documentation/devicetree/bindings/pci/fsl,layerscape-pcie.yaml index daeab5c0758d..3a76d662309d 100644 --- a/Documentation/devicetree/bindings/pci/fsl,layerscape-pcie.yaml +++ b/Documentation/devicetree/bindings/pci/fsl,layerscape-pcie.yaml @@ -69,6 +69,14 @@ properties: minItems: 1 maxItems: 2 + num-viewport: + $ref: /schemas/types.yaml#/definitions/uint32 + deprecated: true + description: + Number of outbound view ports configured in hardware. It's the same as + the number of outbound AT windows. + maximum: 256 + required: - compatible - reg From f66b63ef10d65cf8ce57a7341262308443705628 Mon Sep 17 00:00:00 2001 From: Frank Li Date: Mon, 1 Jul 2024 18:16:12 -0400 Subject: [PATCH 107/302] dt-bindings: PCI: layerscape-pci: Change property 'fsl,pcie-scfg' type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fsl,pcie-scfg requires an argument when there are more than one PCIe instances. Thus, change it to the phandle-array type and use items to describe what each field means. This also fixes the following warning: arch/arm64/boot/dts/freescale/fsl-ls1043a-rdb.dtb: pcie@3400000: fsl,pcie-scfg:0: [22, 0] is too long from schema $id: http://devicetree.org/schemas/pci/fsl,layerscape-pcie.yaml# Link: https://lore.kernel.org/linux-pci/20240701221612.2112668-1-Frank.Li@nxp.com Signed-off-by: Frank Li [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Rob Herring --- .../devicetree/bindings/pci/fsl,layerscape-pcie.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/pci/fsl,layerscape-pcie.yaml b/Documentation/devicetree/bindings/pci/fsl,layerscape-pcie.yaml index 3a76d662309d..be79712836c4 100644 --- a/Documentation/devicetree/bindings/pci/fsl,layerscape-pcie.yaml +++ b/Documentation/devicetree/bindings/pci/fsl,layerscape-pcie.yaml @@ -45,10 +45,15 @@ properties: - const: config fsl,pcie-scfg: - $ref: /schemas/types.yaml#/definitions/phandle + $ref: /schemas/types.yaml#/definitions/phandle-array description: A phandle to the SCFG device node. The second entry is the physical PCIe controller index starting from '0'. This is used to get SCFG PEXN registers. + items: + items: + - description: A phandle to the SCFG device node + - description: PCIe controller index starting from '0' + maxItems: 1 big-endian: $ref: /schemas/types.yaml#/definitions/flag From 364cfd8a56c0eec874057514b8cee220494746f5 Mon Sep 17 00:00:00 2001 From: Rayyan Ansari Date: Mon, 22 Jul 2024 10:18:50 +0100 Subject: [PATCH 108/302] dt-bindings: PCI: qcom,pcie-sc7280: Update bindings adding eight interrupts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previous commit to this bindings, commit 756485bfbb85 ("dt-bindings: PCI: qcom,pcie-sc7280: Move SC7280 to dedicated schema"), updated the bindings to specify one interrupt only, as the devicetree at that time did not describe the hardware fully. The devicetree for SC7280 now specifies eight interrupts, following the commit b8ba66b40da3 ("arm64: dts: qcom: sc7280: Add additional MSI interrupts"). Thus, update the bindings to reflect this. Link: https://lore.kernel.org/linux-pci/20240722-sc7280-pcie-interrupts-v2-1-a5414d3dbc64@linaro.org Signed-off-by: Rayyan Ansari [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Krzysztof Kozlowski Reviewed-by: Manivannan Sadhasivam --- .../bindings/pci/qcom,pcie-sc7280.yaml | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie-sc7280.yaml b/Documentation/devicetree/bindings/pci/qcom,pcie-sc7280.yaml index 634da24ec3ed..5cf1f9165301 100644 --- a/Documentation/devicetree/bindings/pci/qcom,pcie-sc7280.yaml +++ b/Documentation/devicetree/bindings/pci/qcom,pcie-sc7280.yaml @@ -53,11 +53,19 @@ properties: - const: aggre1 # Aggre NoC PCIe1 AXI clock interrupts: - maxItems: 1 + minItems: 8 + maxItems: 8 interrupt-names: items: - - const: msi + - const: msi0 + - const: msi1 + - const: msi2 + - const: msi3 + - const: msi4 + - const: msi5 + - const: msi6 + - const: msi7 resets: maxItems: 1 @@ -137,8 +145,16 @@ examples: dma-coherent; - interrupts = ; - interrupt-names = "msi"; + interrupts = , + , + , + , + , + , + , + ; + interrupt-names = "msi0", "msi1", "msi2", "msi3", + "msi4", "msi5", "msi6", "msi7"; #interrupt-cells = <1>; interrupt-map-mask = <0 0 0 0x7>; interrupt-map = <0 0 0 1 &intc 0 0 0 434 IRQ_TYPE_LEVEL_HIGH>, From b08929e1ec2f69db49e119a0d6af7cf32c813f5e Mon Sep 17 00:00:00 2001 From: Matthew Gerlach Date: Tue, 2 Jul 2024 11:26:52 -0500 Subject: [PATCH 109/302] dt-bindings: PCI: altera: Convert to YAML MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Convert the devicetree bindings for the Altera Root Port PCIe controller from text to YAML. While at it, update the entries in the interrupt-map field to have the correct number of address cells for the interrupt parent. Link: https://lore.kernel.org/linux-pci/20240702162652.1349121-1-matthew.gerlach@linux.intel.com Signed-off-by: Matthew Gerlach [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Conor Dooley --- .../devicetree/bindings/pci/altera-pcie.txt | 50 -------- .../bindings/pci/altr,pcie-root-port.yaml | 114 ++++++++++++++++++ MAINTAINERS | 2 +- 3 files changed, 115 insertions(+), 51 deletions(-) delete mode 100644 Documentation/devicetree/bindings/pci/altera-pcie.txt create mode 100644 Documentation/devicetree/bindings/pci/altr,pcie-root-port.yaml diff --git a/Documentation/devicetree/bindings/pci/altera-pcie.txt b/Documentation/devicetree/bindings/pci/altera-pcie.txt deleted file mode 100644 index 816b244a221e..000000000000 --- a/Documentation/devicetree/bindings/pci/altera-pcie.txt +++ /dev/null @@ -1,50 +0,0 @@ -* Altera PCIe controller - -Required properties: -- compatible : should contain "altr,pcie-root-port-1.0" or "altr,pcie-root-port-2.0" -- reg: a list of physical base address and length for TXS and CRA. - For "altr,pcie-root-port-2.0", additional HIP base address and length. -- reg-names: must include the following entries: - "Txs": TX slave port region - "Cra": Control register access region - "Hip": Hard IP region (if "altr,pcie-root-port-2.0") -- interrupts: specifies the interrupt source of the parent interrupt - controller. The format of the interrupt specifier depends - on the parent interrupt controller. -- device_type: must be "pci" -- #address-cells: set to <3> -- #size-cells: set to <2> -- #interrupt-cells: set to <1> -- ranges: describes the translation of addresses for root ports and - standard PCI regions. -- interrupt-map-mask and interrupt-map: standard PCI properties to define the - mapping of the PCIe interface to interrupt numbers. - -Optional properties: -- msi-parent: Link to the hardware entity that serves as the MSI controller - for this PCIe controller. -- bus-range: PCI bus numbers covered - -Example - pcie_0: pcie@c00000000 { - compatible = "altr,pcie-root-port-1.0"; - reg = <0xc0000000 0x20000000>, - <0xff220000 0x00004000>; - reg-names = "Txs", "Cra"; - interrupt-parent = <&hps_0_arm_gic_0>; - interrupts = <0 40 4>; - interrupt-controller; - #interrupt-cells = <1>; - bus-range = <0x0 0xFF>; - device_type = "pci"; - msi-parent = <&msi_to_gic_gen_0>; - #address-cells = <3>; - #size-cells = <2>; - interrupt-map-mask = <0 0 0 7>; - interrupt-map = <0 0 0 1 &pcie_0 1>, - <0 0 0 2 &pcie_0 2>, - <0 0 0 3 &pcie_0 3>, - <0 0 0 4 &pcie_0 4>; - ranges = <0x82000000 0x00000000 0x00000000 0xc0000000 0x00000000 0x10000000 - 0x82000000 0x00000000 0x10000000 0xd0000000 0x00000000 0x10000000>; - }; diff --git a/Documentation/devicetree/bindings/pci/altr,pcie-root-port.yaml b/Documentation/devicetree/bindings/pci/altr,pcie-root-port.yaml new file mode 100644 index 000000000000..52533fccc134 --- /dev/null +++ b/Documentation/devicetree/bindings/pci/altr,pcie-root-port.yaml @@ -0,0 +1,114 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +# Copyright (C) 2015, 2019, 2024, Intel Corporation +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/altr,pcie-root-port.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Altera PCIe Root Port + +maintainers: + - Matthew Gerlach + +properties: + compatible: + enum: + - altr,pcie-root-port-1.0 + - altr,pcie-root-port-2.0 + + reg: + items: + - description: TX slave port region + - description: Control register access region + - description: Hard IP region + minItems: 2 + + reg-names: + items: + - const: Txs + - const: Cra + - const: Hip + minItems: 2 + + interrupts: + maxItems: 1 + + interrupt-controller: true + + interrupt-map-mask: + items: + - const: 0 + - const: 0 + - const: 0 + - const: 7 + + interrupt-map: + maxItems: 4 + + "#interrupt-cells": + const: 1 + + msi-parent: true + +required: + - compatible + - reg + - reg-names + - interrupts + - "#interrupt-cells" + - interrupt-controller + - interrupt-map + - interrupt-map-mask + +allOf: + - $ref: /schemas/pci/pci-host-bridge.yaml# + - if: + properties: + compatible: + enum: + - altr,pcie-root-port-1.0 + then: + properties: + reg: + maxItems: 2 + + reg-names: + maxItems: 2 + + else: + properties: + reg: + minItems: 3 + + reg-names: + minItems: 3 + + +unevaluatedProperties: false + +examples: + - | + #include + #include + pcie_0: pcie@c00000000 { + compatible = "altr,pcie-root-port-1.0"; + reg = <0xc0000000 0x20000000>, + <0xff220000 0x00004000>; + reg-names = "Txs", "Cra"; + interrupt-parent = <&hps_0_arm_gic_0>; + interrupts = ; + interrupt-controller; + #interrupt-cells = <1>; + bus-range = <0x0 0xff>; + device_type = "pci"; + msi-parent = <&msi_to_gic_gen_0>; + #address-cells = <3>; + #size-cells = <2>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie_0 0 0 0 1>, + <0 0 0 2 &pcie_0 0 0 0 2>, + <0 0 0 3 &pcie_0 0 0 0 3>, + <0 0 0 4 &pcie_0 0 0 0 4>; + ranges = <0x82000000 0x00000000 0x00000000 0xc0000000 0x00000000 0x10000000>, + <0x82000000 0x00000000 0x10000000 0xd0000000 0x00000000 0x10000000>; + }; diff --git a/MAINTAINERS b/MAINTAINERS index 42decde38320..8ebc8a46fc0f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -17372,7 +17372,7 @@ PCI DRIVER FOR ALTERA PCIE IP M: Joyce Ooi L: linux-pci@vger.kernel.org S: Supported -F: Documentation/devicetree/bindings/pci/altera-pcie.txt +F: Documentation/devicetree/bindings/pci/altr,pcie-root-port.yaml F: drivers/pci/controller/pcie-altera.c PCI DRIVER FOR APPLIEDMICRO XGENE From 78efa53e715e21a97c722dba20f8437a0860521e Mon Sep 17 00:00:00 2001 From: Mariusz Tkaczyk Date: Wed, 4 Sep 2024 12:48:46 +0200 Subject: [PATCH 110/302] leds: Init leds class earlier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NPEM driver will require leds class, there is an init-order conflict. Make sure that LEDs initialization happens first and add comment. Link: https://lore.kernel.org/r/20240904104848.23480-2-mariusz.tkaczyk@linux.intel.com Suggested-by: Dan Williams Signed-off-by: Mariusz Tkaczyk Signed-off-by: Bjorn Helgaas Tested-by: Stuart Hayes Reviewed-by: Christoph Hellwig Reviewed-by: Ilpo Järvinen --- drivers/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/Makefile b/drivers/Makefile index fe9ceb0d2288..45d1c3e630f7 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -17,6 +17,9 @@ obj-$(CONFIG_PINCTRL) += pinctrl/ obj-$(CONFIG_GPIOLIB) += gpio/ obj-y += pwm/ +# LEDs must come before PCI, it is needed by NPEM driver +obj-y += leds/ + obj-y += pci/ obj-$(CONFIG_PARISC) += parisc/ @@ -130,7 +133,6 @@ obj-$(CONFIG_CPU_IDLE) += cpuidle/ obj-y += mmc/ obj-y += ufs/ obj-$(CONFIG_MEMSTICK) += memstick/ -obj-y += leds/ obj-$(CONFIG_INFINIBAND) += infiniband/ obj-y += firmware/ obj-$(CONFIG_CRYPTO) += crypto/ From 4e893545ef8712d25f3176790ebb95beb073637e Mon Sep 17 00:00:00 2001 From: Mariusz Tkaczyk Date: Wed, 4 Sep 2024 12:48:47 +0200 Subject: [PATCH 111/302] PCI/NPEM: Add Native PCIe Enclosure Management support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Native PCIe Enclosure Management (NPEM, PCIe r6.1 sec 6.28) allows managing LEDs in storage enclosures. NPEM is indication oriented and it does not give direct access to LEDs. Although each indication *could* represent an individual LED, multiple indications could also be represented as a single, multi-color LED or a single LED blinking in a specific interval. The specification leaves that open. Each enabled indication (capability register bit on) is represented as a ledclass_dev which can be controlled through sysfs. For every ledclass device only 2 brightness states are allowed: LED_ON (1) or LED_OFF (0). This corresponds to the NPEM control register (Indication bit on/off). Ledclass devices appear in sysfs as child devices (subdirectory) of PCI device which has an NPEM Extended Capability and indication is enabled in NPEM capability register. For example, these are LEDs created for pcieport "10000:02:05.0" on my setup: leds/ ├── 10000:02:05.0:enclosure:fail ├── 10000:02:05.0:enclosure:locate ├── 10000:02:05.0:enclosure:ok └── 10000:02:05.0:enclosure:rebuild They can be also found in "/sys/class/leds" directory. The parent PCIe device domain/bus/device/function address is used to guarantee uniqueness across leds subsystem. To enable/disable a "fail" indication, the "brightness" file can be edited: echo 1 > ./leds/10000:02:05.0:enclosure:fail/brightness echo 0 > ./leds/10000:02:05.0:enclosure:fail/brightness PCIe r6.1, sec 7.9.19.2 defines the possible indications. Multiple indications for same parent PCIe device can conflict and hardware may update them when processing new request. To avoid issues, driver refresh all indications by reading back control register. This driver expects to be the exclusive NPEM extended capability manager. It waits up to 1 second after imposing new request, it doesn't verify if controller is busy before write, and it assumes the mutex lock gives protection from concurrent updates. If _DSM LED management is available, we assume the platform may be using NPEM for its own purposes (see PCI Firmware Spec r3.3 sec 4.7), so the driver does not use NPEM. A future patch will add _DSM support; an info message notes whether NPEM or _DSM is being used. NPEM is a PCIe extended capability so it should be registered in pcie_init_capabilities() but it is not possible due to LED dependency. The parent pci_device must be added earlier for led_classdev_register() to be successful. NPEM does not require configuration on kernel side, so it is safe to register LED devices later. Link: https://lore.kernel.org/r/20240904104848.23480-3-mariusz.tkaczyk@linux.intel.com Suggested-by: Lukas Wunner Signed-off-by: Mariusz Tkaczyk Signed-off-by: Bjorn Helgaas Tested-by: Stuart Hayes Reviewed-by: Christoph Hellwig Reviewed-by: Ilpo Järvinen --- Documentation/ABI/testing/sysfs-bus-pci | 63 ++++ drivers/pci/Kconfig | 9 + drivers/pci/Makefile | 1 + drivers/pci/npem.c | 423 ++++++++++++++++++++++++ drivers/pci/pci.h | 8 + drivers/pci/probe.c | 2 + drivers/pci/remove.c | 2 + include/linux/pci.h | 3 + include/uapi/linux/pci_regs.h | 35 ++ 9 files changed, 546 insertions(+) create mode 100644 drivers/pci/npem.c diff --git a/Documentation/ABI/testing/sysfs-bus-pci b/Documentation/ABI/testing/sysfs-bus-pci index ecf47559f495..9ddbb1364698 100644 --- a/Documentation/ABI/testing/sysfs-bus-pci +++ b/Documentation/ABI/testing/sysfs-bus-pci @@ -500,3 +500,66 @@ Description: console drivers from the device. Raw users of pci-sysfs resourceN attributes must be terminated prior to resizing. Success of the resizing operation is not guaranteed. + +What: /sys/bus/pci/devices/.../leds/*:enclosure:*/brightness +What: /sys/class/leds/*:enclosure:*/brightness +Date: August 2024 +KernelVersion: 6.12 +Description: + LED indications on PCIe storage enclosures which are controlled + through the NPEM interface (Native PCIe Enclosure Management, + PCIe r6.1 sec 6.28) are accessible as led class devices, both + below /sys/class/leds and below NPEM-capable PCI devices. + + Although these led class devices could be manipulated manually, + in practice they are typically manipulated automatically by an + application such as ledmon(8). + + The name of a led class device is as follows: + :enclosure: + where: + + - is the domain, bus, device and function number + (e.g. 10000:02:05.0) + - is a short description of the LED indication + + Valid indications per PCIe r6.1 table 6-27 are: + + - ok (drive is functioning normally) + - locate (drive is being identified by an admin) + - fail (drive is not functioning properly) + - rebuild (drive is part of an array that is rebuilding) + - pfa (drive is predicted to fail soon) + - hotspare (drive is marked to be used as a replacement) + - ica (drive is part of an array that is degraded) + - ifa (drive is part of an array that is failed) + - idt (drive is not the right type for the connector) + - disabled (drive is disabled, removal is safe) + - specific0 to specific7 (enclosure-specific indications) + + Broadly, the indications fall into one of these categories: + + - to signify drive state (ok, locate, fail, idt, disabled) + - to signify drive role or state in a software RAID array + (rebuild, pfa, hotspare, ica, ifa) + - to signify any other role or state (specific0 to specific7) + + Mandatory indications per PCIe r6.1 sec 7.9.19.2 comprise: + ok, locate, fail, rebuild. All others are optional. + A led class device is only visible if the corresponding + indication is supported by the device. + + To manipulate the indications, write 0 (LED_OFF) or 1 (LED_ON) + to the "brightness" file. Note that manipulating an indication + may implicitly manipulate other indications at the vendor's + discretion. E.g. when the user lights up the "ok" indication, + the vendor may choose to automatically turn off the "fail" + indication. The current state of an indication can be + retrieved by reading its "brightness" file. + + The PCIe Base Specification allows vendors leeway to choose + different colors or blinking patterns for the indications, + but they typically follow the IBPI standard. E.g. the "locate" + indication is usually presented as one or two LEDs blinking at + 4 Hz frequency: + https://en.wikipedia.org/wiki/International_Blinking_Pattern_Interpretation diff --git a/drivers/pci/Kconfig b/drivers/pci/Kconfig index aa4d1833f442..0d94e4a967d8 100644 --- a/drivers/pci/Kconfig +++ b/drivers/pci/Kconfig @@ -143,6 +143,15 @@ config PCI_IOV If unsure, say N. +config PCI_NPEM + bool "Native PCIe Enclosure Management" + depends on LEDS_CLASS=y + help + Support for Native PCIe Enclosure Management. It allows managing LED + indications in storage enclosures. Enclosure must support following + indications: OK, Locate, Fail, Rebuild, other indications are + optional. + config PCI_PRI bool "PCI PRI support" select PCI_ATS diff --git a/drivers/pci/Makefile b/drivers/pci/Makefile index 8ddad57934a6..374c5c06d92f 100644 --- a/drivers/pci/Makefile +++ b/drivers/pci/Makefile @@ -35,6 +35,7 @@ obj-$(CONFIG_XEN_PCIDEV_FRONTEND) += xen-pcifront.o obj-$(CONFIG_VGA_ARB) += vgaarb.o obj-$(CONFIG_PCI_DOE) += doe.o obj-$(CONFIG_PCI_DYNAMIC_OF_NODES) += of_property.o +obj-$(CONFIG_PCI_NPEM) += npem.o # Endpoint library must be initialized before its users obj-$(CONFIG_PCI_ENDPOINT) += endpoint/ diff --git a/drivers/pci/npem.c b/drivers/pci/npem.c new file mode 100644 index 000000000000..5fe33d145108 --- /dev/null +++ b/drivers/pci/npem.c @@ -0,0 +1,423 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe Enclosure management driver created for LED interfaces based on + * indications. It says *what indications* blink but does not specify *how* + * they blink - it is hardware defined. + * + * The driver name refers to Native PCIe Enclosure Management. It is + * first indication oriented standard with specification. + * + * Native PCIe Enclosure Management (NPEM) + * PCIe Base Specification r6.1 sec 6.28, 7.9.19 + * + * Copyright (c) 2023-2024 Intel Corporation + * Mariusz Tkaczyk + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "pci.h" + +struct indication { + u32 bit; + const char *name; +}; + +static const struct indication npem_indications[] = { + {PCI_NPEM_IND_OK, "enclosure:ok"}, + {PCI_NPEM_IND_LOCATE, "enclosure:locate"}, + {PCI_NPEM_IND_FAIL, "enclosure:fail"}, + {PCI_NPEM_IND_REBUILD, "enclosure:rebuild"}, + {PCI_NPEM_IND_PFA, "enclosure:pfa"}, + {PCI_NPEM_IND_HOTSPARE, "enclosure:hotspare"}, + {PCI_NPEM_IND_ICA, "enclosure:ica"}, + {PCI_NPEM_IND_IFA, "enclosure:ifa"}, + {PCI_NPEM_IND_IDT, "enclosure:idt"}, + {PCI_NPEM_IND_DISABLED, "enclosure:disabled"}, + {PCI_NPEM_IND_SPEC_0, "enclosure:specific_0"}, + {PCI_NPEM_IND_SPEC_1, "enclosure:specific_1"}, + {PCI_NPEM_IND_SPEC_2, "enclosure:specific_2"}, + {PCI_NPEM_IND_SPEC_3, "enclosure:specific_3"}, + {PCI_NPEM_IND_SPEC_4, "enclosure:specific_4"}, + {PCI_NPEM_IND_SPEC_5, "enclosure:specific_5"}, + {PCI_NPEM_IND_SPEC_6, "enclosure:specific_6"}, + {PCI_NPEM_IND_SPEC_7, "enclosure:specific_7"}, + {0, NULL} +}; + +#define for_each_indication(ind, inds) \ + for (ind = inds; ind->bit; ind++) + +/* + * The driver has internal list of supported indications. Ideally, the driver + * should not touch bits that are not defined and for which LED devices are + * not exposed but in reality, it needs to turn them off. + * + * Otherwise, there will be no possibility to turn off indications turned on by + * other utilities or turned on by default and it leads to bad user experience. + * + * Additionally, it excludes NPEM commands like RESET or ENABLE. + */ +static u32 reg_to_indications(u32 caps, const struct indication *inds) +{ + const struct indication *ind; + u32 supported_indications = 0; + + for_each_indication(ind, inds) + supported_indications |= ind->bit; + + return caps & supported_indications; +} + +/** + * struct npem_led - LED details + * @indication: indication details + * @npem: NPEM device + * @name: LED name + * @led: LED device + */ +struct npem_led { + const struct indication *indication; + struct npem *npem; + char name[LED_MAX_NAME_SIZE]; + struct led_classdev led; +}; + +/** + * struct npem_ops - backend specific callbacks + * @get_active_indications: get active indications + * npem: NPEM device + * inds: response buffer + * @set_active_indications: set new indications + * npem: npem device + * inds: bit mask to set + * @inds: supported indications array, set of indications is backend specific + * @name: backend name + */ +struct npem_ops { + int (*get_active_indications)(struct npem *npem, u32 *inds); + int (*set_active_indications)(struct npem *npem, u32 inds); + const struct indication *inds; + const char *name; +}; + +/** + * struct npem - NPEM device properties + * @dev: PCI device this driver is attached to + * @ops: backend specific callbacks + * @lock: serializes concurrent access to NPEM device by multiple LED devices + * @pos: cached offset of NPEM Capability Register in Configuration Space; + * only used if NPEM registers are accessed directly and not through _DSM + * @supported_indications: cached bit mask of supported indications; + * non-indication and reserved bits in the NPEM Capability Register are + * cleared in this bit mask + * @active_indications: cached bit mask of active indications; + * non-indication and reserved bits in the NPEM Control Register are + * cleared in this bit mask + * @led_cnt: size of @leds array + * @leds: array containing LED class devices of all supported LEDs + */ +struct npem { + struct pci_dev *dev; + const struct npem_ops *ops; + struct mutex lock; + u16 pos; + u32 supported_indications; + u32 active_indications; + int led_cnt; + struct npem_led leds[]; +}; + +static int npem_read_reg(struct npem *npem, u16 reg, u32 *val) +{ + int ret = pci_read_config_dword(npem->dev, npem->pos + reg, val); + + return pcibios_err_to_errno(ret); +} + +static int npem_write_ctrl(struct npem *npem, u32 reg) +{ + int pos = npem->pos + PCI_NPEM_CTRL; + int ret = pci_write_config_dword(npem->dev, pos, reg); + + return pcibios_err_to_errno(ret); +} + +static int npem_get_active_indications(struct npem *npem, u32 *inds) +{ + u32 ctrl; + int ret; + + ret = npem_read_reg(npem, PCI_NPEM_CTRL, &ctrl); + if (ret) + return ret; + + /* If PCI_NPEM_CTRL_ENABLE is not set then no indication should blink */ + if (!(ctrl & PCI_NPEM_CTRL_ENABLE)) { + *inds = 0; + return 0; + } + + *inds = ctrl & npem->supported_indications; + + return 0; +} + +static int npem_set_active_indications(struct npem *npem, u32 inds) +{ + int ctrl, ret, ret_val; + u32 cc_status; + + lockdep_assert_held(&npem->lock); + + /* This bit is always required */ + ctrl = inds | PCI_NPEM_CTRL_ENABLE; + + ret = npem_write_ctrl(npem, ctrl); + if (ret) + return ret; + + /* + * For the case where a NPEM command has not completed immediately, + * it is recommended that software not continuously "spin" on polling + * the status register, but rather poll under interrupt at a reduced + * rate; for example at 10 ms intervals. + * + * PCIe r6.1 sec 6.28 "Implementation Note: Software Polling of NPEM + * Command Completed" + */ + ret = read_poll_timeout(npem_read_reg, ret_val, + ret_val || (cc_status & PCI_NPEM_STATUS_CC), + 10 * USEC_PER_MSEC, USEC_PER_SEC, false, npem, + PCI_NPEM_STATUS, &cc_status); + if (ret) + return ret; + if (ret_val) + return ret_val; + + /* + * All writes to control register, including writes that do not change + * the register value, are NPEM commands and should eventually result + * in a command completion indication in the NPEM Status Register. + * + * PCIe Base Specification r6.1 sec 7.9.19.3 + * + * Register may not be updated, or other conflicting bits may be + * cleared. Spec is not strict here. Read NPEM Control register after + * write to keep cache in-sync. + */ + return npem_get_active_indications(npem, &npem->active_indications); +} + +static const struct npem_ops npem_ops = { + .get_active_indications = npem_get_active_indications, + .set_active_indications = npem_set_active_indications, + .name = "Native PCIe Enclosure Management", + .inds = npem_indications, +}; + +#define DSM_GUID GUID_INIT(0x5d524d9d, 0xfff9, 0x4d4b, 0x8c, 0xb7, 0x74, 0x7e,\ + 0xd5, 0x1e, 0x19, 0x4d) +#define GET_SUPPORTED_STATES_DSM 1 +#define GET_STATE_DSM 2 +#define SET_STATE_DSM 3 + +static const guid_t dsm_guid = DSM_GUID; + +static bool npem_has_dsm(struct pci_dev *pdev) +{ + acpi_handle handle; + + handle = ACPI_HANDLE(&pdev->dev); + if (!handle) + return false; + + return acpi_check_dsm(handle, &dsm_guid, 0x1, + BIT(GET_SUPPORTED_STATES_DSM) | + BIT(GET_STATE_DSM) | BIT(SET_STATE_DSM)); +} + +/* + * The status of each indicator is cached on first brightness_ get/set time + * and updated at write time. brightness_get() is only responsible for + * reflecting the last written/cached value. + */ +static enum led_brightness brightness_get(struct led_classdev *led) +{ + struct npem_led *nled = container_of(led, struct npem_led, led); + struct npem *npem = nled->npem; + int ret, val = 0; + + ret = mutex_lock_interruptible(&npem->lock); + if (ret) + return ret; + + if (npem->active_indications & nled->indication->bit) + val = 1; + + mutex_unlock(&npem->lock); + return val; +} + +static int brightness_set(struct led_classdev *led, + enum led_brightness brightness) +{ + struct npem_led *nled = container_of(led, struct npem_led, led); + struct npem *npem = nled->npem; + u32 indications; + int ret; + + ret = mutex_lock_interruptible(&npem->lock); + if (ret) + return ret; + + if (brightness == 0) + indications = npem->active_indications & ~(nled->indication->bit); + else + indications = npem->active_indications | nled->indication->bit; + + ret = npem->ops->set_active_indications(npem, indications); + + mutex_unlock(&npem->lock); + return ret; +} + +static void npem_free(struct npem *npem) +{ + struct npem_led *nled; + int cnt; + + if (!npem) + return; + + for (cnt = 0; cnt < npem->led_cnt; cnt++) { + nled = &npem->leds[cnt]; + + if (nled->name[0]) + led_classdev_unregister(&nled->led); + } + + mutex_destroy(&npem->lock); + kfree(npem); +} + +static int pci_npem_set_led_classdev(struct npem *npem, struct npem_led *nled) +{ + struct led_classdev *led = &nled->led; + struct led_init_data init_data = {}; + char *name = nled->name; + int ret; + + init_data.devicename = pci_name(npem->dev); + init_data.default_label = nled->indication->name; + + ret = led_compose_name(&npem->dev->dev, &init_data, name); + if (ret) + return ret; + + led->name = name; + led->brightness_set_blocking = brightness_set; + led->brightness_get = brightness_get; + led->max_brightness = 1; + led->default_trigger = "none"; + led->flags = 0; + + ret = led_classdev_register(&npem->dev->dev, led); + if (ret) + /* Clear the name to indicate that it is not registered. */ + name[0] = 0; + return ret; +} + +static int pci_npem_init(struct pci_dev *dev, const struct npem_ops *ops, + int pos, u32 caps) +{ + u32 supported = reg_to_indications(caps, ops->inds); + int supported_cnt = hweight32(supported); + const struct indication *indication; + struct npem_led *nled; + struct npem *npem; + int led_idx = 0; + int ret; + + npem = kzalloc(struct_size(npem, leds, supported_cnt), GFP_KERNEL); + if (!npem) + return -ENOMEM; + + npem->supported_indications = supported; + npem->led_cnt = supported_cnt; + npem->pos = pos; + npem->dev = dev; + npem->ops = ops; + + ret = npem->ops->get_active_indications(npem, + &npem->active_indications); + if (ret) + return ret; + + mutex_init(&npem->lock); + + for_each_indication(indication, npem_indications) { + if (!(npem->supported_indications & indication->bit)) + continue; + + nled = &npem->leds[led_idx++]; + nled->indication = indication; + nled->npem = npem; + + ret = pci_npem_set_led_classdev(npem, nled); + if (ret) { + npem_free(npem); + return ret; + } + } + + dev->npem = npem; + return 0; +} + +void pci_npem_remove(struct pci_dev *dev) +{ + npem_free(dev->npem); +} + +void pci_npem_create(struct pci_dev *dev) +{ + const struct npem_ops *ops = &npem_ops; + int pos = 0, ret; + u32 cap; + + if (npem_has_dsm(dev)) { + /* + * OS should use the DSM for LED control if it is available + * PCI Firmware Spec r3.3 sec 4.7. + */ + pci_info(dev, "Not configuring %s because _DSM is present\n", + ops->name); + return; + } else { + pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_NPEM); + if (pos == 0) + return; + + if (pci_read_config_dword(dev, pos + PCI_NPEM_CAP, &cap) != 0 || + (cap & PCI_NPEM_CAP_CAPABLE) == 0) + return; + } + + pci_info(dev, "Configuring %s\n", ops->name); + + ret = pci_npem_init(dev, ops, pos, cap); + if (ret) + pci_err(dev, "Failed to register %s, err: %d\n", ops->name, + ret); +} diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 79c8398f3938..554fd9dfe25c 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -398,6 +398,14 @@ static inline void pci_doe_destroy(struct pci_dev *pdev) { } static inline void pci_doe_disconnected(struct pci_dev *pdev) { } #endif +#ifdef CONFIG_PCI_NPEM +void pci_npem_create(struct pci_dev *dev); +void pci_npem_remove(struct pci_dev *dev); +#else +static inline void pci_npem_create(struct pci_dev *dev) { } +static inline void pci_npem_remove(struct pci_dev *dev) { } +#endif + /** * pci_dev_set_io_state - Set the new error state if possible. * diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index b14b9876c030..17ee559c31a4 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -2593,6 +2593,8 @@ void pci_device_add(struct pci_dev *dev, struct pci_bus *bus) dev->match_driver = false; ret = device_add(&dev->dev); WARN_ON(ret < 0); + + pci_npem_create(dev); } struct pci_dev *pci_scan_single_device(struct pci_bus *bus, int devfn) diff --git a/drivers/pci/remove.c b/drivers/pci/remove.c index 910387e5bdbf..da9629c3a688 100644 --- a/drivers/pci/remove.c +++ b/drivers/pci/remove.c @@ -34,6 +34,8 @@ static void pci_destroy_dev(struct pci_dev *dev) if (!dev->dev.kobj.parent) return; + pci_npem_remove(dev); + device_del(&dev->dev); down_write(&pci_bus_sem); diff --git a/include/linux/pci.h b/include/linux/pci.h index 4cf89a4b4cbc..c9db853e269f 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -516,6 +516,9 @@ struct pci_dev { #endif #ifdef CONFIG_PCI_DOE struct xarray doe_mbs; /* Data Object Exchange mailboxes */ +#endif +#ifdef CONFIG_PCI_NPEM + struct npem *npem; /* Native PCIe Enclosure Management */ #endif u16 acs_cap; /* ACS Capability offset */ phys_addr_t rom; /* Physical address if not from BAR */ diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h index 94c00996e633..9751b413f3d6 100644 --- a/include/uapi/linux/pci_regs.h +++ b/include/uapi/linux/pci_regs.h @@ -740,6 +740,7 @@ #define PCI_EXT_CAP_ID_DVSEC 0x23 /* Designated Vendor-Specific */ #define PCI_EXT_CAP_ID_DLF 0x25 /* Data Link Feature */ #define PCI_EXT_CAP_ID_PL_16GT 0x26 /* Physical Layer 16.0 GT/s */ +#define PCI_EXT_CAP_ID_NPEM 0x29 /* Native PCIe Enclosure Management */ #define PCI_EXT_CAP_ID_PL_32GT 0x2A /* Physical Layer 32.0 GT/s */ #define PCI_EXT_CAP_ID_DOE 0x2E /* Data Object Exchange */ #define PCI_EXT_CAP_ID_MAX PCI_EXT_CAP_ID_DOE @@ -1121,6 +1122,40 @@ #define PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK 0x000000F0 #define PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT 4 +/* Native PCIe Enclosure Management */ +#define PCI_NPEM_CAP 0x04 /* NPEM capability register */ +#define PCI_NPEM_CAP_CAPABLE 0x00000001 /* NPEM Capable */ + +#define PCI_NPEM_CTRL 0x08 /* NPEM control register */ +#define PCI_NPEM_CTRL_ENABLE 0x00000001 /* NPEM Enable */ + +/* + * Native PCIe Enclosure Management indication bits and Reset command bit + * are corresponding for capability and control registers. + */ +#define PCI_NPEM_CMD_RESET 0x00000002 /* Reset Command */ +#define PCI_NPEM_IND_OK 0x00000004 /* OK */ +#define PCI_NPEM_IND_LOCATE 0x00000008 /* Locate */ +#define PCI_NPEM_IND_FAIL 0x00000010 /* Fail */ +#define PCI_NPEM_IND_REBUILD 0x00000020 /* Rebuild */ +#define PCI_NPEM_IND_PFA 0x00000040 /* Predicted Failure Analysis */ +#define PCI_NPEM_IND_HOTSPARE 0x00000080 /* Hot Spare */ +#define PCI_NPEM_IND_ICA 0x00000100 /* In Critical Array */ +#define PCI_NPEM_IND_IFA 0x00000200 /* In Failed Array */ +#define PCI_NPEM_IND_IDT 0x00000400 /* Device Type */ +#define PCI_NPEM_IND_DISABLED 0x00000800 /* Disabled */ +#define PCI_NPEM_IND_SPEC_0 0x01000000 +#define PCI_NPEM_IND_SPEC_1 0x02000000 +#define PCI_NPEM_IND_SPEC_2 0x04000000 +#define PCI_NPEM_IND_SPEC_3 0x08000000 +#define PCI_NPEM_IND_SPEC_4 0x10000000 +#define PCI_NPEM_IND_SPEC_5 0x20000000 +#define PCI_NPEM_IND_SPEC_6 0x40000000 +#define PCI_NPEM_IND_SPEC_7 0x80000000 + +#define PCI_NPEM_STATUS 0x0c /* NPEM status register */ +#define PCI_NPEM_STATUS_CC 0x00000001 /* Command Completed */ + /* Data Object Exchange */ #define PCI_DOE_CAP 0x04 /* DOE Capabilities Register */ #define PCI_DOE_CAP_INT_SUP 0x00000001 /* Interrupt Support */ From 8745aaab60a63ff57311e9a8dda5dfb0b3a33907 Mon Sep 17 00:00:00 2001 From: Jinjie Ruan Date: Wed, 28 Aug 2024 18:42:02 +0800 Subject: [PATCH 112/302] PCI: altera: Replace TLP_REQ_ID() with macro PCI_DEVID() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The TLP_REQ_ID's function is same as current PCI_DEVID() macro, replace it. No functional changes intended. Link: https://lore.kernel.org/linux-pci/20240828104202.3683491-1-ruanjinjie@huawei.com Signed-off-by: Jinjie Ruan Signed-off-by: Krzysztof Wilczyński Reviewed-by: Manivannan Sadhasivam --- drivers/pci/controller/pcie-altera.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/pci/controller/pcie-altera.c b/drivers/pci/controller/pcie-altera.c index ef73baefaeb9..650b2dd81c48 100644 --- a/drivers/pci/controller/pcie-altera.c +++ b/drivers/pci/controller/pcie-altera.c @@ -55,12 +55,11 @@ #define TLP_READ_TAG 0x1d #define TLP_WRITE_TAG 0x10 #define RP_DEVFN 0 -#define TLP_REQ_ID(bus, devfn) (((bus) << 8) | (devfn)) #define TLP_CFG_DW0(pcie, cfg) \ (((cfg) << 24) | \ TLP_PAYLOAD_SIZE) #define TLP_CFG_DW1(pcie, tag, be) \ - (((TLP_REQ_ID(pcie->root_bus_nr, RP_DEVFN)) << 16) | (tag << 8) | (be)) + (((PCI_DEVID(pcie->root_bus_nr, RP_DEVFN)) << 16) | (tag << 8) | (be)) #define TLP_CFG_DW2(bus, devfn, offset) \ (((bus) << 24) | ((devfn) << 16) | (offset)) #define TLP_COMP_STATUS(s) (((s) >> 13) & 7) From c500a86693a126c9393e602741e348f80f1b0fc5 Mon Sep 17 00:00:00 2001 From: Alexandra Diupina Date: Tue, 3 Sep 2024 14:58:23 +0300 Subject: [PATCH 113/302] PCI: kirin: Fix buffer overflow in kirin_pcie_parse_port() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Within kirin_pcie_parse_port(), the pcie->num_slots is compared to pcie->gpio_id_reset size (MAX_PCI_SLOTS) which is correct and would lead to an overflow. Thus, fix condition to pcie->num_slots + 1 >= MAX_PCI_SLOTS and move pcie->num_slots increment below the if-statement to avoid out-of-bounds array access. Found by Linux Verification Center (linuxtesting.org) with SVACE. Fixes: b22dbbb24571 ("PCI: kirin: Support PERST# GPIOs for HiKey970 external PEX 8606 bridge") Link: https://lore.kernel.org/linux-pci/20240903115823.30647-1-adiupina@astralinux.ru Signed-off-by: Alexandra Diupina [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Mauro Carvalho Chehab --- drivers/pci/controller/dwc/pcie-kirin.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/pci/controller/dwc/pcie-kirin.c b/drivers/pci/controller/dwc/pcie-kirin.c index 0a29136491b8..85a2c77b1835 100644 --- a/drivers/pci/controller/dwc/pcie-kirin.c +++ b/drivers/pci/controller/dwc/pcie-kirin.c @@ -420,11 +420,11 @@ static int kirin_pcie_parse_port(struct kirin_pcie *pcie, "unable to get a valid reset gpio\n"); } - pcie->num_slots++; - if (pcie->num_slots > MAX_PCI_SLOTS) { + if (pcie->num_slots + 1 >= MAX_PCI_SLOTS) { dev_err(dev, "Too many PCI slots!\n"); return -EINVAL; } + pcie->num_slots++; ret = of_pci_get_devfn(child); if (ret < 0) { From 026f84d3fa62d215b11cbeb5a5d97df941e93b5c Mon Sep 17 00:00:00 2001 From: Subramanian Ananthanarayanan Date: Fri, 6 Sep 2024 10:52:27 +0530 Subject: [PATCH 114/302] PCI: Add ACS quirk for Qualcomm SA8775P MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Qualcomm SA8775P root ports don't advertise an ACS capability, but they do provide ACS-like features to disable peer transactions and validate bus numbers in requests. Thus, add an ACS quirk for the SA8775P. Link: https://lore.kernel.org/linux-pci/20240906052228.1829485-1-quic_skananth@quicinc.com Signed-off-by: Subramanian Ananthanarayanan Signed-off-by: Krzysztof Wilczyński --- drivers/pci/quirks.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index a2ce4e08edf5..e085ec3d9982 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -5070,6 +5070,8 @@ static const struct pci_dev_acs_enabled { /* QCOM QDF2xxx root ports */ { PCI_VENDOR_ID_QCOM, 0x0400, pci_quirk_qcom_rp_acs }, { PCI_VENDOR_ID_QCOM, 0x0401, pci_quirk_qcom_rp_acs }, + /* QCOM SA8775P root port */ + { PCI_VENDOR_ID_QCOM, 0x0115, pci_quirk_qcom_rp_acs }, /* HXT SD4800 root ports. The ACS design is same as QCOM QDF2xxx */ { PCI_VENDOR_ID_HXT, 0x0401, pci_quirk_qcom_rp_acs }, /* Intel PCH root ports */ From 759ec28242894f2006a1606c1d6e9aca48cecfcf Mon Sep 17 00:00:00 2001 From: Mariusz Tkaczyk Date: Wed, 4 Sep 2024 12:48:48 +0200 Subject: [PATCH 115/302] PCI/NPEM: Add _DSM PCIe SSD status LED management MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The PCIe SSD Status LED Management _DSM defined in PCI Firmware Spec r3.3 sec 4.7 provides a way to manage LEDs via ACPI. The design is similar to NPEM defined in PCIe Base Specification r6.1 sec 6.28: - Both standards are indication oriented, - _DSM supported bits correspond to NPEM capability register bits, - _DSM control bits correspond to NPEM control register bits. _DSM does not support enclosure-specific indications or the special NPEM commands NPEM_ENABLE and NPEM_RESET. _DSM is implemented as a second backend in NPEM driver. The backend used is logged with info priority. The same sysfs interface is used for both NPEM and _DSM. According to spec, _DSM has higher priority, and availability of _DSM in not limited to devices with NPEM support. The Dell implementation of DSM uses acpi ipmi, which may not be available immediately (in fact it may take up to 10s for this interface to be available). It can determine if DSM is supported (GET_SUPPORTED_STATES_DSM is working) but it cannot serve GET_STATE_DSM or SET_STATE_DSM commands in this time. From userspace application perspective (primarily configured by systemd service) it is better to have not working but configured interface rather than have it available after few seconds. For that reason, npem->active_indications cache is now loaded lazily, i.e. any GET or SET request want cache to be updated if it is not done yet. Link: https://lore.kernel.org/r/20240904104848.23480-4-mariusz.tkaczyk@linux.intel.com Suggested-by: Lukas Wunner Signed-off-by: Stuart Hayes Signed-off-by: Mariusz Tkaczyk Signed-off-by: Bjorn Helgaas Tested-by: Stuart Hayes Reviewed-by: Christoph Hellwig Reviewed-by: Ilpo Järvinen --- Documentation/ABI/testing/sysfs-bus-pci | 9 ++ drivers/pci/npem.c | 188 +++++++++++++++++++++++- 2 files changed, 189 insertions(+), 8 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-bus-pci b/Documentation/ABI/testing/sysfs-bus-pci index 9ddbb1364698..7f63c7e97773 100644 --- a/Documentation/ABI/testing/sysfs-bus-pci +++ b/Documentation/ABI/testing/sysfs-bus-pci @@ -563,3 +563,12 @@ Description: indication is usually presented as one or two LEDs blinking at 4 Hz frequency: https://en.wikipedia.org/wiki/International_Blinking_Pattern_Interpretation + + PCI Firmware Specification r3.3 sec 4.7 defines a DSM interface + to facilitate shared access by operating system and platform + firmware to a device's NPEM registers. The kernel will use + this DSM interface where available, instead of accessing NPEM + registers directly. The DSM interface does not support the + enclosure-specific indications "specific0" to "specific7", + hence the corresponding led class devices are unavailable if + the DSM interface is used. diff --git a/drivers/pci/npem.c b/drivers/pci/npem.c index 5fe33d145108..97507e0df769 100644 --- a/drivers/pci/npem.c +++ b/drivers/pci/npem.c @@ -10,6 +10,14 @@ * Native PCIe Enclosure Management (NPEM) * PCIe Base Specification r6.1 sec 6.28, 7.9.19 * + * _DSM Definitions for PCIe SSD Status LED + * PCI Firmware Specification, r3.3 sec 4.7 + * + * Two backends are supported to manipulate indications: Direct NPEM register + * access (npem_ops) and indirect access through the ACPI _DSM (dsm_ops). + * _DSM is used if supported, else NPEM. + * + * Copyright (c) 2021-2022 Dell Inc. * Copyright (c) 2023-2024 Intel Corporation * Mariusz Tkaczyk */ @@ -54,6 +62,21 @@ static const struct indication npem_indications[] = { {0, NULL} }; +/* _DSM PCIe SSD LED States correspond to NPEM register values */ +static const struct indication dsm_indications[] = { + {PCI_NPEM_IND_OK, "enclosure:ok"}, + {PCI_NPEM_IND_LOCATE, "enclosure:locate"}, + {PCI_NPEM_IND_FAIL, "enclosure:fail"}, + {PCI_NPEM_IND_REBUILD, "enclosure:rebuild"}, + {PCI_NPEM_IND_PFA, "enclosure:pfa"}, + {PCI_NPEM_IND_HOTSPARE, "enclosure:hotspare"}, + {PCI_NPEM_IND_ICA, "enclosure:ica"}, + {PCI_NPEM_IND_IFA, "enclosure:ifa"}, + {PCI_NPEM_IND_IDT, "enclosure:idt"}, + {PCI_NPEM_IND_DISABLED, "enclosure:disabled"}, + {0, NULL} +}; + #define for_each_indication(ind, inds) \ for (ind = inds; ind->bit; ind++) @@ -123,6 +146,14 @@ struct npem_ops { * @active_indications: cached bit mask of active indications; * non-indication and reserved bits in the NPEM Control Register are * cleared in this bit mask + * @active_inds_initialized: whether @active_indications has been initialized; + * On Dell platforms, it is required that IPMI drivers are loaded before + * the GET_STATE_DSM method is invoked: They use an IPMI OpRegion to + * get/set the active LEDs. By initializing @active_indications lazily + * (on first access to an LED), IPMI drivers are given a chance to load. + * If they are not loaded in time, users will see various errors on LED + * access in dmesg. Once they are loaded, the errors go away and LED + * access becomes possible. * @led_cnt: size of @leds array * @leds: array containing LED class devices of all supported LEDs */ @@ -133,6 +164,7 @@ struct npem { u16 pos; u32 supported_indications; u32 active_indications; + unsigned int active_inds_initialized:1; int led_cnt; struct npem_led leds[]; }; @@ -246,6 +278,139 @@ static bool npem_has_dsm(struct pci_dev *pdev) BIT(GET_STATE_DSM) | BIT(SET_STATE_DSM)); } +struct dsm_output { + u16 status; + u8 function_specific_err; + u8 vendor_specific_err; + u32 state; +}; + +/** + * dsm_evaluate() - send DSM PCIe SSD Status LED command + * @pdev: PCI device + * @dsm_func: DSM LED Function + * @output: buffer to copy DSM Response + * @value_to_set: value for SET_STATE_DSM function + * + * To not bother caller with ACPI context, the returned _DSM Output Buffer is + * copied. + */ +static int dsm_evaluate(struct pci_dev *pdev, u64 dsm_func, + struct dsm_output *output, u32 value_to_set) +{ + acpi_handle handle = ACPI_HANDLE(&pdev->dev); + union acpi_object *out_obj, arg3[2]; + union acpi_object *arg3_p = NULL; + + if (dsm_func == SET_STATE_DSM) { + arg3[0].type = ACPI_TYPE_PACKAGE; + arg3[0].package.count = 1; + arg3[0].package.elements = &arg3[1]; + + arg3[1].type = ACPI_TYPE_BUFFER; + arg3[1].buffer.length = 4; + arg3[1].buffer.pointer = (u8 *)&value_to_set; + + arg3_p = arg3; + } + + out_obj = acpi_evaluate_dsm_typed(handle, &dsm_guid, 0x1, dsm_func, + arg3_p, ACPI_TYPE_BUFFER); + if (!out_obj) + return -EIO; + + if (out_obj->buffer.length < sizeof(struct dsm_output)) { + ACPI_FREE(out_obj); + return -EIO; + } + + memcpy(output, out_obj->buffer.pointer, sizeof(struct dsm_output)); + + ACPI_FREE(out_obj); + return 0; +} + +static int dsm_get(struct pci_dev *pdev, u64 dsm_func, u32 *buf) +{ + struct dsm_output output; + int ret = dsm_evaluate(pdev, dsm_func, &output, 0); + + if (ret) + return ret; + + if (output.status != 0) + return -EIO; + + *buf = output.state; + return 0; +} + +static int dsm_get_active_indications(struct npem *npem, u32 *buf) +{ + int ret = dsm_get(npem->dev, GET_STATE_DSM, buf); + + /* Filter out not supported indications in response */ + *buf &= npem->supported_indications; + return ret; +} + +static int dsm_set_active_indications(struct npem *npem, u32 value) +{ + struct dsm_output output; + int ret = dsm_evaluate(npem->dev, SET_STATE_DSM, &output, value); + + if (ret) + return ret; + + switch (output.status) { + case 4: + /* + * Not all bits are set. If this bit is set, the platform + * disregarded some or all of the request state changes. OSPM + * should check the resulting PCIe SSD Status LED States to see + * what, if anything, has changed. + * + * PCI Firmware Specification, r3.3 Table 4-19. + */ + if (output.function_specific_err != 1) + return -EIO; + fallthrough; + case 0: + break; + default: + return -EIO; + } + + npem->active_indications = output.state; + + return 0; +} + +static const struct npem_ops dsm_ops = { + .get_active_indications = dsm_get_active_indications, + .set_active_indications = dsm_set_active_indications, + .name = "_DSM PCIe SSD Status LED Management", + .inds = dsm_indications, +}; + +static int npem_initialize_active_indications(struct npem *npem) +{ + int ret; + + lockdep_assert_held(&npem->lock); + + if (npem->active_inds_initialized) + return 0; + + ret = npem->ops->get_active_indications(npem, + &npem->active_indications); + if (ret) + return ret; + + npem->active_inds_initialized = true; + return 0; +} + /* * The status of each indicator is cached on first brightness_ get/set time * and updated at write time. brightness_get() is only responsible for @@ -261,9 +426,14 @@ static enum led_brightness brightness_get(struct led_classdev *led) if (ret) return ret; + ret = npem_initialize_active_indications(npem); + if (ret) + goto out; + if (npem->active_indications & nled->indication->bit) val = 1; +out: mutex_unlock(&npem->lock); return val; } @@ -280,6 +450,10 @@ static int brightness_set(struct led_classdev *led, if (ret) return ret; + ret = npem_initialize_active_indications(npem); + if (ret) + goto out; + if (brightness == 0) indications = npem->active_indications & ~(nled->indication->bit); else @@ -287,6 +461,7 @@ static int brightness_set(struct led_classdev *led, ret = npem->ops->set_active_indications(npem, indications); +out: mutex_unlock(&npem->lock); return ret; } @@ -359,11 +534,6 @@ static int pci_npem_init(struct pci_dev *dev, const struct npem_ops *ops, npem->dev = dev; npem->ops = ops; - ret = npem->ops->get_active_indications(npem, - &npem->active_indications); - if (ret) - return ret; - mutex_init(&npem->lock); for_each_indication(indication, npem_indications) { @@ -401,9 +571,11 @@ void pci_npem_create(struct pci_dev *dev) * OS should use the DSM for LED control if it is available * PCI Firmware Spec r3.3 sec 4.7. */ - pci_info(dev, "Not configuring %s because _DSM is present\n", - ops->name); - return; + ret = dsm_get(dev, GET_SUPPORTED_STATES_DSM, &cap); + if (ret) + return; + + ops = &dsm_ops; } else { pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_NPEM); if (pos == 0) From ae6476c6de187bea90c729e3e0188143300fa671 Mon Sep 17 00:00:00 2001 From: Jim Quinlan Date: Thu, 15 Aug 2024 18:57:23 -0400 Subject: [PATCH 116/302] PCI: brcmstb: Refactor for chips with many regular inbound windows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Provide support for new chips with multiple inbound windows while keeping the legacy support for the older chips. In existing chips there are three inbound windows with fixed purposes: the first was for mapping SoC internal registers, the second was for memory, and the third was for memory but with the endian swapped. Typically, only one window was used. Complicating the inbound window usage was the fact that the PCIe HW would do a baroque internal mapping of system memory, and concatenate the regions of multiple memory controllers. Newer chips such as the 7712 and Cable Modem SoCs take a step forward and drop the internal mapping while providing for multiple inbound windows. This works in concert with the dma-ranges property, where each provided range becomes an inbound window. Link: https://lore.kernel.org/linux-pci/20240815225731.40276-11-james.quinlan@broadcom.com Co-developed-by: Riyan Dhiman Signed-off-by: Riyan Dhiman Signed-off-by: Jim Quinlan [kwilczynski: commit log, wrap code comments to 80 columns] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Florian Fainelli Reviewed-by: Stanimir Varbanov Acked-by: Manivannan Sadhasivam --- drivers/pci/controller/pcie-brcmstb.c | 236 ++++++++++++++++++++------ 1 file changed, 182 insertions(+), 54 deletions(-) diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/controller/pcie-brcmstb.c index f1d890aea369..9bbef10fed0d 100644 --- a/drivers/pci/controller/pcie-brcmstb.c +++ b/drivers/pci/controller/pcie-brcmstb.c @@ -75,15 +75,19 @@ #define PCIE_MEM_WIN0_HI(win) \ PCIE_MISC_CPU_2_PCIE_MEM_WIN0_HI + ((win) * 8) +/* + * NOTE: You may see the term "BAR" in a number of register names used by + * this driver. The term is an artifact of when the HW core was an + * endpoint device (EP). Now it is a root complex (RC) and anywhere a + * register has the term "BAR" it is related to an inbound window. + */ + +#define PCIE_BRCM_MAX_INBOUND_WINS 16 #define PCIE_MISC_RC_BAR1_CONFIG_LO 0x402c #define PCIE_MISC_RC_BAR1_CONFIG_LO_SIZE_MASK 0x1f -#define PCIE_MISC_RC_BAR2_CONFIG_LO 0x4034 -#define PCIE_MISC_RC_BAR2_CONFIG_LO_SIZE_MASK 0x1f -#define PCIE_MISC_RC_BAR2_CONFIG_HI 0x4038 +#define PCIE_MISC_RC_BAR4_CONFIG_LO 0x40d4 -#define PCIE_MISC_RC_BAR3_CONFIG_LO 0x403c -#define PCIE_MISC_RC_BAR3_CONFIG_LO_SIZE_MASK 0x1f #define PCIE_MISC_MSI_BAR_CONFIG_LO 0x4044 #define PCIE_MISC_MSI_BAR_CONFIG_HI 0x4048 @@ -130,6 +134,10 @@ (PCIE_MISC_HARD_PCIE_HARD_DEBUG_CLKREQ_DEBUG_ENABLE_MASK | \ PCIE_MISC_HARD_PCIE_HARD_DEBUG_L1SS_ENABLE_MASK) +#define PCIE_MISC_UBUS_BAR1_CONFIG_REMAP 0x40ac +#define PCIE_MISC_UBUS_BAR1_CONFIG_REMAP_ACCESS_EN_MASK BIT(0) +#define PCIE_MISC_UBUS_BAR4_CONFIG_REMAP 0x410c + #define PCIE_MSI_INTR2_BASE 0x4500 /* Offsets from INTR2_CPU and MSI_INTR2 BASE offsets */ @@ -217,12 +225,20 @@ enum pcie_type { BCM4908, BCM7278, BCM2711, + BCM7712, +}; + +struct inbound_win { + u64 size; + u64 pci_offset; + u64 cpu_addr; }; struct pcie_cfg_data { const int *offsets; const enum pcie_type type; const bool has_phy; + u8 num_inbound_wins; void (*perst_set)(struct brcm_pcie *pcie, u32 val); void (*bridge_sw_init_set)(struct brcm_pcie *pcie, u32 val); }; @@ -274,6 +290,7 @@ struct brcm_pcie { struct subdev_regulators *sr; bool ep_wakeup_capable; bool has_phy; + u8 num_inbound_wins; }; static inline bool is_bmips(const struct brcm_pcie *pcie) @@ -396,7 +413,7 @@ static void brcm_pcie_set_gen(struct brcm_pcie *pcie, int gen) } static void brcm_pcie_set_outbound_win(struct brcm_pcie *pcie, - unsigned int win, u64 cpu_addr, + u8 win, u64 cpu_addr, u64 pcie_addr, u64 size) { u32 cpu_addr_mb_high, limit_addr_mb_high; @@ -791,23 +808,62 @@ static void brcm_pcie_perst_set_generic(struct brcm_pcie *pcie, u32 val) writel(tmp, pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); } -static int brcm_pcie_get_rc_bar2_size_and_offset(struct brcm_pcie *pcie, - u64 *rc_bar2_size, - u64 *rc_bar2_offset) +static void add_inbound_win(struct inbound_win *b, u8 *count, u64 size, + u64 cpu_addr, u64 pci_offset) +{ + b->size = size; + b->cpu_addr = cpu_addr; + b->pci_offset = pci_offset; + (*count)++; +} + +static int brcm_pcie_get_inbound_wins(struct brcm_pcie *pcie, + struct inbound_win inbound_wins[]) { struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + u64 pci_offset, cpu_addr, size = 0, tot_size = 0; struct resource_entry *entry; struct device *dev = pcie->dev; u64 lowest_pcie_addr = ~(u64)0; int ret, i = 0; - u64 size = 0; + u8 n = 0; + + /* + * The HW registers (and PCIe) use order-1 numbering for BARs. As such, + * we have inbound_wins[0] unused and BAR1 starts at inbound_wins[1]. + */ + struct inbound_win *b_begin = &inbound_wins[1]; + struct inbound_win *b = b_begin; + + /* + * STB chips beside 7712 disable the first inbound window default. + * Rather being mapped to system memory it is mapped to the + * internal registers of the SoC. This feature is deprecated, has + * security considerations, and is not implemented in our modern + * SoCs. + */ + if (pcie->type != BCM7712) + add_inbound_win(b++, &n, 0, 0, 0); resource_list_for_each_entry(entry, &bridge->dma_ranges) { - u64 pcie_beg = entry->res->start - entry->offset; + u64 pcie_start = entry->res->start - entry->offset; + u64 cpu_start = entry->res->start; - size += entry->res->end - entry->res->start + 1; - if (pcie_beg < lowest_pcie_addr) - lowest_pcie_addr = pcie_beg; + size = resource_size(entry->res); + tot_size += size; + if (pcie_start < lowest_pcie_addr) + lowest_pcie_addr = pcie_start; + /* + * 7712 and newer chips may have many BARs, with each + * offering a non-overlapping viewport to system memory. + * That being said, each BARs size must still be a power of + * two. + */ + if (pcie->type == BCM7712) + add_inbound_win(b++, &n, size, cpu_start, pcie_start); + + if (n > pcie->num_inbound_wins) + break; } if (lowest_pcie_addr == ~(u64)0) { @@ -815,13 +871,20 @@ static int brcm_pcie_get_rc_bar2_size_and_offset(struct brcm_pcie *pcie, return -EINVAL; } + /* + * 7712 and newer chips do not have an internal memory mapping system + * that enables multiple memory controllers. As such, it can return + * now w/o doing special configuration. + */ + if (pcie->type == BCM7712) + return n; + ret = of_property_read_variable_u64_array(pcie->np, "brcm,scb-sizes", pcie->memc_size, 1, PCIE_BRCM_MAX_MEMC); - if (ret <= 0) { /* Make an educated guess */ pcie->num_memc = 1; - pcie->memc_size[0] = 1ULL << fls64(size - 1); + pcie->memc_size[0] = 1ULL << fls64(tot_size - 1); } else { pcie->num_memc = ret; } @@ -830,10 +893,15 @@ static int brcm_pcie_get_rc_bar2_size_and_offset(struct brcm_pcie *pcie, for (i = 0, size = 0; i < pcie->num_memc; i++) size += pcie->memc_size[i]; - /* System memory starts at this address in PCIe-space */ - *rc_bar2_offset = lowest_pcie_addr; - /* The sum of all memc views must also be a power of 2 */ - *rc_bar2_size = 1ULL << fls64(size - 1); + /* Our HW mandates that the window size must be a power of 2 */ + size = 1ULL << fls64(size - 1); + + /* + * For STB chips, the BAR2 cpu_addr is hardwired to the start + * of system memory, so we set it to 0. + */ + cpu_addr = 0; + pci_offset = lowest_pcie_addr; /* * We validate the inbound memory view even though we should trust @@ -868,25 +936,91 @@ static int brcm_pcie_get_rc_bar2_size_and_offset(struct brcm_pcie *pcie, * outbound memory @ 3GB). So instead it will start at the 1x * multiple of its size */ - if (!*rc_bar2_size || (*rc_bar2_offset & (*rc_bar2_size - 1)) || - (*rc_bar2_offset < SZ_4G && *rc_bar2_offset > SZ_2G)) { - dev_err(dev, "Invalid rc_bar2_offset/size: size 0x%llx, off 0x%llx\n", - *rc_bar2_size, *rc_bar2_offset); + if (!size || (pci_offset & (size - 1)) || + (pci_offset < SZ_4G && pci_offset > SZ_2G)) { + dev_err(dev, "Invalid inbound_win2_offset/size: size 0x%llx, off 0x%llx\n", + size, pci_offset); return -EINVAL; } - return 0; + /* Enable inbound window 2, the main inbound window for STB chips */ + add_inbound_win(b++, &n, size, cpu_addr, pci_offset); + + /* + * Disable inbound window 3. On some chips presents the same + * window as #2 but the data appears in a settable endianness. + */ + add_inbound_win(b++, &n, 0, 0, 0); + + return n; +} + +static u32 brcm_bar_reg_offset(int bar) +{ + if (bar <= 3) + return PCIE_MISC_RC_BAR1_CONFIG_LO + 8 * (bar - 1); + else + return PCIE_MISC_RC_BAR4_CONFIG_LO + 8 * (bar - 4); +} + +static u32 brcm_ubus_reg_offset(int bar) +{ + if (bar <= 3) + return PCIE_MISC_UBUS_BAR1_CONFIG_REMAP + 8 * (bar - 1); + else + return PCIE_MISC_UBUS_BAR4_CONFIG_REMAP + 8 * (bar - 4); +} + +static void set_inbound_win_registers(struct brcm_pcie *pcie, + const struct inbound_win *inbound_wins, + u8 num_inbound_wins) +{ + void __iomem *base = pcie->base; + int i; + + for (i = 1; i <= num_inbound_wins; i++) { + u64 pci_offset = inbound_wins[i].pci_offset; + u64 cpu_addr = inbound_wins[i].cpu_addr; + u64 size = inbound_wins[i].size; + u32 reg_offset = brcm_bar_reg_offset(i); + u32 tmp = lower_32_bits(pci_offset); + + u32p_replace_bits(&tmp, brcm_pcie_encode_ibar_size(size), + PCIE_MISC_RC_BAR1_CONFIG_LO_SIZE_MASK); + + /* Write low */ + writel_relaxed(tmp, base + reg_offset); + /* Write high */ + writel_relaxed(upper_32_bits(pci_offset), base + reg_offset + 4); + + /* + * Most STB chips: + * Do nothing. + * 7712: + * All of their BARs need to be set. + */ + if (pcie->type == BCM7712) { + /* BUS remap register settings */ + reg_offset = brcm_ubus_reg_offset(i); + tmp = lower_32_bits(cpu_addr) & ~0xfff; + tmp |= PCIE_MISC_UBUS_BAR1_CONFIG_REMAP_ACCESS_EN_MASK; + writel_relaxed(tmp, base + reg_offset); + tmp = upper_32_bits(cpu_addr); + writel_relaxed(tmp, base + reg_offset + 4); + } + } } static int brcm_pcie_setup(struct brcm_pcie *pcie) { - u64 rc_bar2_offset, rc_bar2_size; + struct inbound_win inbound_wins[PCIE_BRCM_MAX_INBOUND_WINS]; void __iomem *base = pcie->base; struct pci_host_bridge *bridge; struct resource_entry *entry; u32 tmp, burst, aspm_support; - int num_out_wins = 0; - int ret, memc; + u8 num_out_wins = 0; + int num_inbound_wins = 0; + int memc; /* Reset the bridge */ pcie->bridge_sw_init_set(pcie, 1); @@ -935,17 +1069,16 @@ static int brcm_pcie_setup(struct brcm_pcie *pcie) u32p_replace_bits(&tmp, 1, PCIE_MISC_MISC_CTRL_PCIE_RCB_64B_MODE_MASK); writel(tmp, base + PCIE_MISC_MISC_CTRL); - ret = brcm_pcie_get_rc_bar2_size_and_offset(pcie, &rc_bar2_size, - &rc_bar2_offset); - if (ret) - return ret; + num_inbound_wins = brcm_pcie_get_inbound_wins(pcie, inbound_wins); + if (num_inbound_wins < 0) + return num_inbound_wins; - tmp = lower_32_bits(rc_bar2_offset); - u32p_replace_bits(&tmp, brcm_pcie_encode_ibar_size(rc_bar2_size), - PCIE_MISC_RC_BAR2_CONFIG_LO_SIZE_MASK); - writel(tmp, base + PCIE_MISC_RC_BAR2_CONFIG_LO); - writel(upper_32_bits(rc_bar2_offset), - base + PCIE_MISC_RC_BAR2_CONFIG_HI); + set_inbound_win_registers(pcie, inbound_wins, num_inbound_wins); + + if (!brcm_pcie_rc_mode(pcie)) { + dev_err(pcie->dev, "PCIe RC controller misconfigured as Endpoint\n"); + return -EINVAL; + } tmp = readl(base + PCIE_MISC_MISC_CTRL); for (memc = 0; memc < pcie->num_memc; memc++) { @@ -967,25 +1100,12 @@ static int brcm_pcie_setup(struct brcm_pcie *pcie) * 4GB or when the inbound area is smaller than 4GB (taking into * account the rounding-up we're forced to perform). */ - if (rc_bar2_offset >= SZ_4G || (rc_bar2_size + rc_bar2_offset) < SZ_4G) + if (inbound_wins[2].pci_offset >= SZ_4G || + (inbound_wins[2].size + inbound_wins[2].pci_offset) < SZ_4G) pcie->msi_target_addr = BRCM_MSI_TARGET_ADDR_LT_4GB; else pcie->msi_target_addr = BRCM_MSI_TARGET_ADDR_GT_4GB; - if (!brcm_pcie_rc_mode(pcie)) { - dev_err(pcie->dev, "PCIe RC controller misconfigured as Endpoint\n"); - return -EINVAL; - } - - /* disable the PCIe->GISB memory window (RC_BAR1) */ - tmp = readl(base + PCIE_MISC_RC_BAR1_CONFIG_LO); - tmp &= ~PCIE_MISC_RC_BAR1_CONFIG_LO_SIZE_MASK; - writel(tmp, base + PCIE_MISC_RC_BAR1_CONFIG_LO); - - /* disable the PCIe->SCB memory window (RC_BAR3) */ - tmp = readl(base + PCIE_MISC_RC_BAR3_CONFIG_LO); - tmp &= ~PCIE_MISC_RC_BAR3_CONFIG_LO_SIZE_MASK; - writel(tmp, base + PCIE_MISC_RC_BAR3_CONFIG_LO); /* Don't advertise L0s capability if 'aspm-no-l0s' */ aspm_support = PCIE_LINK_STATE_L1; @@ -1036,7 +1156,7 @@ static int brcm_pcie_setup(struct brcm_pcie *pcie) num_out_wins++; } - /* PCIe->SCB endian mode for BAR */ + /* PCIe->SCB endian mode for inbound window */ tmp = readl(base + PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1); u32p_replace_bits(&tmp, PCIE_RC_CFG_VENDOR_SPCIFIC_REG1_LITTLE_ENDIAN, PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1_ENDIAN_MODE_BAR2_MASK); @@ -1518,6 +1638,7 @@ static const struct pcie_cfg_data generic_cfg = { .type = GENERIC, .perst_set = brcm_pcie_perst_set_generic, .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, + .num_inbound_wins = 3, }; static const struct pcie_cfg_data bcm7425_cfg = { @@ -1525,6 +1646,7 @@ static const struct pcie_cfg_data bcm7425_cfg = { .type = BCM7425, .perst_set = brcm_pcie_perst_set_generic, .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, + .num_inbound_wins = 3, }; static const struct pcie_cfg_data bcm7435_cfg = { @@ -1532,6 +1654,7 @@ static const struct pcie_cfg_data bcm7435_cfg = { .type = BCM7435, .perst_set = brcm_pcie_perst_set_generic, .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, + .num_inbound_wins = 3, }; static const struct pcie_cfg_data bcm4908_cfg = { @@ -1539,6 +1662,7 @@ static const struct pcie_cfg_data bcm4908_cfg = { .type = BCM4908, .perst_set = brcm_pcie_perst_set_4908, .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, + .num_inbound_wins = 3, }; static const int pcie_offset_bcm7278[] = { @@ -1554,6 +1678,7 @@ static const struct pcie_cfg_data bcm7278_cfg = { .type = BCM7278, .perst_set = brcm_pcie_perst_set_7278, .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_7278, + .num_inbound_wins = 3, }; static const struct pcie_cfg_data bcm2711_cfg = { @@ -1561,6 +1686,7 @@ static const struct pcie_cfg_data bcm2711_cfg = { .type = BCM2711, .perst_set = brcm_pcie_perst_set_generic, .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, + .num_inbound_wins = 3, }; static const struct pcie_cfg_data bcm7216_cfg = { @@ -1569,6 +1695,7 @@ static const struct pcie_cfg_data bcm7216_cfg = { .perst_set = brcm_pcie_perst_set_7278, .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_7278, .has_phy = true, + .num_inbound_wins = 3, }; static const struct of_device_id brcm_pcie_match[] = { @@ -1625,6 +1752,7 @@ static int brcm_pcie_probe(struct platform_device *pdev) pcie->perst_set = data->perst_set; pcie->bridge_sw_init_set = data->bridge_sw_init_set; pcie->has_phy = data->has_phy; + pcie->num_inbound_wins = data->num_inbound_wins; pcie->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pcie->base)) From 6f61062fce866e2ff1f3300b278e93d939a846a0 Mon Sep 17 00:00:00 2001 From: Jim Quinlan Date: Thu, 15 Aug 2024 18:57:24 -0400 Subject: [PATCH 117/302] PCI: brcmstb: Check return value of all reset_control_* calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Always check the return value for invocations of reset_control_xxx() and propagate the error to the next level. Although the current functions in reset-brcmstb.c cannot fail, this may someday change. Link: https://lore.kernel.org/linux-pci/20240815225731.40276-12-james.quinlan@broadcom.com Signed-off-by: Jim Quinlan [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Florian Fainelli Reviewed-by: Manivannan Sadhasivam Reviewed-by: Stanimir Varbanov --- drivers/pci/controller/pcie-brcmstb.c | 103 ++++++++++++++++++-------- 1 file changed, 73 insertions(+), 30 deletions(-) diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/controller/pcie-brcmstb.c index 9bbef10fed0d..ffb79c40e925 100644 --- a/drivers/pci/controller/pcie-brcmstb.c +++ b/drivers/pci/controller/pcie-brcmstb.c @@ -239,8 +239,8 @@ struct pcie_cfg_data { const enum pcie_type type; const bool has_phy; u8 num_inbound_wins; - void (*perst_set)(struct brcm_pcie *pcie, u32 val); - void (*bridge_sw_init_set)(struct brcm_pcie *pcie, u32 val); + int (*perst_set)(struct brcm_pcie *pcie, u32 val); + int (*bridge_sw_init_set)(struct brcm_pcie *pcie, u32 val); }; struct subdev_regulators { @@ -285,8 +285,8 @@ struct brcm_pcie { int num_memc; u64 memc_size[PCIE_BRCM_MAX_MEMC]; u32 hw_rev; - void (*perst_set)(struct brcm_pcie *pcie, u32 val); - void (*bridge_sw_init_set)(struct brcm_pcie *pcie, u32 val); + int (*perst_set)(struct brcm_pcie *pcie, u32 val); + int (*bridge_sw_init_set)(struct brcm_pcie *pcie, u32 val); struct subdev_regulators *sr; bool ep_wakeup_capable; bool has_phy; @@ -749,26 +749,33 @@ static void __iomem *brcm7425_pcie_map_bus(struct pci_bus *bus, return base + DATA_ADDR(pcie); } -static void brcm_pcie_bridge_sw_init_set_generic(struct brcm_pcie *pcie, u32 val) +static int brcm_pcie_bridge_sw_init_set_generic(struct brcm_pcie *pcie, u32 val) { u32 tmp, mask = RGR1_SW_INIT_1_INIT_GENERIC_MASK; u32 shift = RGR1_SW_INIT_1_INIT_GENERIC_SHIFT; + int ret = 0; if (pcie->bridge_reset) { if (val) - reset_control_assert(pcie->bridge_reset); + ret = reset_control_assert(pcie->bridge_reset); else - reset_control_deassert(pcie->bridge_reset); + ret = reset_control_deassert(pcie->bridge_reset); - return; + if (ret) + dev_err(pcie->dev, "failed to %s 'bridge' reset, err=%d\n", + val ? "assert" : "deassert", ret); + + return ret; } tmp = readl(pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); tmp = (tmp & ~mask) | ((val << shift) & mask); writel(tmp, pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); + + return ret; } -static void brcm_pcie_bridge_sw_init_set_7278(struct brcm_pcie *pcie, u32 val) +static int brcm_pcie_bridge_sw_init_set_7278(struct brcm_pcie *pcie, u32 val) { u32 tmp, mask = RGR1_SW_INIT_1_INIT_7278_MASK; u32 shift = RGR1_SW_INIT_1_INIT_7278_SHIFT; @@ -776,20 +783,29 @@ static void brcm_pcie_bridge_sw_init_set_7278(struct brcm_pcie *pcie, u32 val) tmp = readl(pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); tmp = (tmp & ~mask) | ((val << shift) & mask); writel(tmp, pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); + + return 0; } -static void brcm_pcie_perst_set_4908(struct brcm_pcie *pcie, u32 val) +static int brcm_pcie_perst_set_4908(struct brcm_pcie *pcie, u32 val) { + int ret; + if (WARN_ONCE(!pcie->perst_reset, "missing PERST# reset controller\n")) - return; + return -EINVAL; if (val) - reset_control_assert(pcie->perst_reset); + ret = reset_control_assert(pcie->perst_reset); else - reset_control_deassert(pcie->perst_reset); + ret = reset_control_deassert(pcie->perst_reset); + + if (ret) + dev_err(pcie->dev, "failed to %s 'perst' reset, err=%d\n", + val ? "assert" : "deassert", ret); + return ret; } -static void brcm_pcie_perst_set_7278(struct brcm_pcie *pcie, u32 val) +static int brcm_pcie_perst_set_7278(struct brcm_pcie *pcie, u32 val) { u32 tmp; @@ -797,15 +813,19 @@ static void brcm_pcie_perst_set_7278(struct brcm_pcie *pcie, u32 val) tmp = readl(pcie->base + PCIE_MISC_PCIE_CTRL); u32p_replace_bits(&tmp, !val, PCIE_MISC_PCIE_CTRL_PCIE_PERSTB_MASK); writel(tmp, pcie->base + PCIE_MISC_PCIE_CTRL); + + return 0; } -static void brcm_pcie_perst_set_generic(struct brcm_pcie *pcie, u32 val) +static int brcm_pcie_perst_set_generic(struct brcm_pcie *pcie, u32 val) { u32 tmp; tmp = readl(pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); u32p_replace_bits(&tmp, val, PCIE_RGR1_SW_INIT_1_PERST_MASK); writel(tmp, pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); + + return 0; } static void add_inbound_win(struct inbound_win *b, u8 *count, u64 size, @@ -1020,19 +1040,28 @@ static int brcm_pcie_setup(struct brcm_pcie *pcie) u32 tmp, burst, aspm_support; u8 num_out_wins = 0; int num_inbound_wins = 0; - int memc; + int memc, ret; /* Reset the bridge */ - pcie->bridge_sw_init_set(pcie, 1); + ret = pcie->bridge_sw_init_set(pcie, 1); + if (ret) + return ret; /* Ensure that PERST# is asserted; some bootloaders may deassert it. */ - if (pcie->type == BCM2711) - pcie->perst_set(pcie, 1); + if (pcie->type == BCM2711) { + ret = pcie->perst_set(pcie, 1); + if (ret) { + pcie->bridge_sw_init_set(pcie, 0); + return ret; + } + } usleep_range(100, 200); /* Take the bridge out of reset */ - pcie->bridge_sw_init_set(pcie, 0); + ret = pcie->bridge_sw_init_set(pcie, 0); + if (ret) + return ret; tmp = readl(base + HARD_DEBUG(pcie)); if (is_bmips(pcie)) @@ -1251,7 +1280,9 @@ static int brcm_pcie_start_link(struct brcm_pcie *pcie) int ret, i; /* Unassert the fundamental reset */ - pcie->perst_set(pcie, 0); + ret = pcie->perst_set(pcie, 0); + if (ret) + return ret; /* * Wait for 100ms after PERST# deassertion; see PCIe CEM specification @@ -1443,15 +1474,17 @@ static inline int brcm_phy_stop(struct brcm_pcie *pcie) return pcie->has_phy ? brcm_phy_cntl(pcie, 0) : 0; } -static void brcm_pcie_turn_off(struct brcm_pcie *pcie) +static int brcm_pcie_turn_off(struct brcm_pcie *pcie) { void __iomem *base = pcie->base; - int tmp; + int tmp, ret; if (brcm_pcie_link_up(pcie)) brcm_pcie_enter_l23(pcie); /* Assert fundamental reset */ - pcie->perst_set(pcie, 1); + ret = pcie->perst_set(pcie, 1); + if (ret) + return ret; /* Deassert request for L23 in case it was asserted */ tmp = readl(base + PCIE_MISC_PCIE_CTRL); @@ -1464,7 +1497,9 @@ static void brcm_pcie_turn_off(struct brcm_pcie *pcie) writel(tmp, base + HARD_DEBUG(pcie)); /* Shutdown PCIe bridge */ - pcie->bridge_sw_init_set(pcie, 1); + ret = pcie->bridge_sw_init_set(pcie, 1); + + return ret; } static int pci_dev_may_wakeup(struct pci_dev *dev, void *data) @@ -1482,9 +1517,12 @@ static int brcm_pcie_suspend_noirq(struct device *dev) { struct brcm_pcie *pcie = dev_get_drvdata(dev); struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); - int ret; + int ret, rret; + + ret = brcm_pcie_turn_off(pcie); + if (ret) + return ret; - brcm_pcie_turn_off(pcie); /* * If brcm_phy_stop() returns an error, just dev_err(). If we * return the error it will cause the suspend to fail and this is a @@ -1513,7 +1551,10 @@ static int brcm_pcie_suspend_noirq(struct device *dev) pcie->sr->supplies); if (ret) { dev_err(dev, "Could not turn off regulators\n"); - reset_control_reset(pcie->rescal); + rret = reset_control_reset(pcie->rescal); + if (rret) + dev_err(dev, "failed to reset 'rascal' controller ret=%d\n", + rret); return ret; } } @@ -1528,7 +1569,7 @@ static int brcm_pcie_resume_noirq(struct device *dev) struct brcm_pcie *pcie = dev_get_drvdata(dev); void __iomem *base; u32 tmp; - int ret; + int ret, rret; base = pcie->base; ret = clk_prepare_enable(pcie->clk); @@ -1590,7 +1631,9 @@ err_regulator: if (pcie->sr) regulator_bulk_disable(pcie->sr->num_supplies, pcie->sr->supplies); err_reset: - reset_control_rearm(pcie->rescal); + rret = reset_control_rearm(pcie->rescal); + if (rret) + dev_err(pcie->dev, "failed to rearm 'rescal' reset, err=%d\n", rret); err_disable_clk: clk_disable_unprepare(pcie->clk); return ret; From 8215851c74f9eb5a9e028834db0ab03d3a6e47e4 Mon Sep 17 00:00:00 2001 From: Jim Quinlan Date: Thu, 15 Aug 2024 18:57:25 -0400 Subject: [PATCH 118/302] PCI: brcmstb: Change field name from 'type' to 'soc_base' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 'type' field used in the driver to discern SoC differences is confusing; change it to the more apt 'soc_base'. The 'base' is because some SoCs have the same characteristics as previous SoCs so it is convenient to classify them in the same group. Link: https://lore.kernel.org/linux-pci/20240815225731.40276-13-james.quinlan@broadcom.com Signed-off-by: Jim Quinlan [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Florian Fainelli Reviewed-by: Manivannan Sadhasivam --- drivers/pci/controller/pcie-brcmstb.c | 42 +++++++++++++-------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/controller/pcie-brcmstb.c index ffb79c40e925..aecb66c29722 100644 --- a/drivers/pci/controller/pcie-brcmstb.c +++ b/drivers/pci/controller/pcie-brcmstb.c @@ -218,7 +218,7 @@ enum { PCIE_INTR2_CPU_BASE, }; -enum pcie_type { +enum pcie_soc_base { GENERIC, BCM7425, BCM7435, @@ -236,7 +236,7 @@ struct inbound_win { struct pcie_cfg_data { const int *offsets; - const enum pcie_type type; + const enum pcie_soc_base soc_base; const bool has_phy; u8 num_inbound_wins; int (*perst_set)(struct brcm_pcie *pcie, u32 val); @@ -277,7 +277,7 @@ struct brcm_pcie { u64 msi_target_addr; struct brcm_msi *msi; const int *reg_offsets; - enum pcie_type type; + enum pcie_soc_base soc_base; struct reset_control *rescal; struct reset_control *perst_reset; struct reset_control *bridge_reset; @@ -295,7 +295,7 @@ struct brcm_pcie { static inline bool is_bmips(const struct brcm_pcie *pcie) { - return pcie->type == BCM7435 || pcie->type == BCM7425; + return pcie->soc_base == BCM7435 || pcie->soc_base == BCM7425; } /* @@ -862,7 +862,7 @@ static int brcm_pcie_get_inbound_wins(struct brcm_pcie *pcie, * security considerations, and is not implemented in our modern * SoCs. */ - if (pcie->type != BCM7712) + if (pcie->soc_base != BCM7712) add_inbound_win(b++, &n, 0, 0, 0); resource_list_for_each_entry(entry, &bridge->dma_ranges) { @@ -879,7 +879,7 @@ static int brcm_pcie_get_inbound_wins(struct brcm_pcie *pcie, * That being said, each BARs size must still be a power of * two. */ - if (pcie->type == BCM7712) + if (pcie->soc_base == BCM7712) add_inbound_win(b++, &n, size, cpu_start, pcie_start); if (n > pcie->num_inbound_wins) @@ -896,7 +896,7 @@ static int brcm_pcie_get_inbound_wins(struct brcm_pcie *pcie, * that enables multiple memory controllers. As such, it can return * now w/o doing special configuration. */ - if (pcie->type == BCM7712) + if (pcie->soc_base == BCM7712) return n; ret = of_property_read_variable_u64_array(pcie->np, "brcm,scb-sizes", pcie->memc_size, 1, @@ -1019,7 +1019,7 @@ static void set_inbound_win_registers(struct brcm_pcie *pcie, * 7712: * All of their BARs need to be set. */ - if (pcie->type == BCM7712) { + if (pcie->soc_base == BCM7712) { /* BUS remap register settings */ reg_offset = brcm_ubus_reg_offset(i); tmp = lower_32_bits(cpu_addr) & ~0xfff; @@ -1048,7 +1048,7 @@ static int brcm_pcie_setup(struct brcm_pcie *pcie) return ret; /* Ensure that PERST# is asserted; some bootloaders may deassert it. */ - if (pcie->type == BCM2711) { + if (pcie->soc_base == BCM2711) { ret = pcie->perst_set(pcie, 1); if (ret) { pcie->bridge_sw_init_set(pcie, 0); @@ -1079,9 +1079,9 @@ static int brcm_pcie_setup(struct brcm_pcie *pcie) */ if (is_bmips(pcie)) burst = 0x1; /* 256 bytes */ - else if (pcie->type == BCM2711) + else if (pcie->soc_base == BCM2711) burst = 0x0; /* 128 bytes */ - else if (pcie->type == BCM7278) + else if (pcie->soc_base == BCM7278) burst = 0x3; /* 512 bytes */ else burst = 0x2; /* 512 bytes */ @@ -1678,7 +1678,7 @@ static const int pcie_offsets_bmips_7425[] = { static const struct pcie_cfg_data generic_cfg = { .offsets = pcie_offsets, - .type = GENERIC, + .soc_base = GENERIC, .perst_set = brcm_pcie_perst_set_generic, .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, .num_inbound_wins = 3, @@ -1686,7 +1686,7 @@ static const struct pcie_cfg_data generic_cfg = { static const struct pcie_cfg_data bcm7425_cfg = { .offsets = pcie_offsets_bmips_7425, - .type = BCM7425, + .soc_base = BCM7425, .perst_set = brcm_pcie_perst_set_generic, .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, .num_inbound_wins = 3, @@ -1694,7 +1694,7 @@ static const struct pcie_cfg_data bcm7425_cfg = { static const struct pcie_cfg_data bcm7435_cfg = { .offsets = pcie_offsets, - .type = BCM7435, + .soc_base = BCM7435, .perst_set = brcm_pcie_perst_set_generic, .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, .num_inbound_wins = 3, @@ -1702,7 +1702,7 @@ static const struct pcie_cfg_data bcm7435_cfg = { static const struct pcie_cfg_data bcm4908_cfg = { .offsets = pcie_offsets, - .type = BCM4908, + .soc_base = BCM4908, .perst_set = brcm_pcie_perst_set_4908, .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, .num_inbound_wins = 3, @@ -1718,7 +1718,7 @@ static const int pcie_offset_bcm7278[] = { static const struct pcie_cfg_data bcm7278_cfg = { .offsets = pcie_offset_bcm7278, - .type = BCM7278, + .soc_base = BCM7278, .perst_set = brcm_pcie_perst_set_7278, .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_7278, .num_inbound_wins = 3, @@ -1726,7 +1726,7 @@ static const struct pcie_cfg_data bcm7278_cfg = { static const struct pcie_cfg_data bcm2711_cfg = { .offsets = pcie_offsets, - .type = BCM2711, + .soc_base = BCM2711, .perst_set = brcm_pcie_perst_set_generic, .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, .num_inbound_wins = 3, @@ -1734,7 +1734,7 @@ static const struct pcie_cfg_data bcm2711_cfg = { static const struct pcie_cfg_data bcm7216_cfg = { .offsets = pcie_offset_bcm7278, - .type = BCM7278, + .soc_base = BCM7278, .perst_set = brcm_pcie_perst_set_7278, .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_7278, .has_phy = true, @@ -1791,7 +1791,7 @@ static int brcm_pcie_probe(struct platform_device *pdev) pcie->dev = &pdev->dev; pcie->np = np; pcie->reg_offsets = data->offsets; - pcie->type = data->type; + pcie->soc_base = data->soc_base; pcie->perst_set = data->perst_set; pcie->bridge_sw_init_set = data->bridge_sw_init_set; pcie->has_phy = data->has_phy; @@ -1869,7 +1869,7 @@ static int brcm_pcie_probe(struct platform_device *pdev) goto fail; pcie->hw_rev = readl(pcie->base + PCIE_MISC_REVISION); - if (pcie->type == BCM4908 && pcie->hw_rev >= BRCM_PCIE_HW_REV_3_20) { + if (pcie->soc_base == BCM4908 && pcie->hw_rev >= BRCM_PCIE_HW_REV_3_20) { dev_err(pcie->dev, "hardware revision with unsupported PERST# setup\n"); ret = -ENODEV; goto fail; @@ -1884,7 +1884,7 @@ static int brcm_pcie_probe(struct platform_device *pdev) } } - bridge->ops = pcie->type == BCM7425 ? &brcm7425_pcie_ops : &brcm_pcie_ops; + bridge->ops = pcie->soc_base == BCM7425 ? &brcm7425_pcie_ops : &brcm_pcie_ops; bridge->sysdata = pcie; platform_set_drvdata(pdev, pcie); From 91e5d15c7b198ecea27407e04cff2fed2d4c2c75 Mon Sep 17 00:00:00 2001 From: Jim Quinlan Date: Thu, 15 Aug 2024 18:57:26 -0400 Subject: [PATCH 119/302] PCI: brcmstb: Enable 7712 SoCs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Broadcom STB 7712 is the sibling chip of the RPi 5 (2712). It has one PCIe controller with a single port, supports gen2 and one lane only. The current revision of the chip is "C0" or "C1". Link: https://lore.kernel.org/linux-pci/20240815225731.40276-14-james.quinlan@broadcom.com Signed-off-by: Jim Quinlan [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Florian Fainelli Reviewed-by: Krzysztof Kozlowski Reviewed-by: Manivannan Sadhasivam Reviewed-by: Stanimir Varbanov --- drivers/pci/controller/pcie-brcmstb.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/controller/pcie-brcmstb.c index aecb66c29722..2bd8ff0f875c 100644 --- a/drivers/pci/controller/pcie-brcmstb.c +++ b/drivers/pci/controller/pcie-brcmstb.c @@ -1205,6 +1205,10 @@ static void brcm_extend_rbus_timeout(struct brcm_pcie *pcie) const unsigned int REG_OFFSET = PCIE_RGR1_SW_INIT_1(pcie) - 8; u32 timeout_us = 4000000; /* 4 seconds, our setting for L1SS */ + /* 7712 does not have this (RGR1) timer */ + if (pcie->soc_base == BCM7712) + return; + /* Each unit in timeout register is 1/216,000,000 seconds */ writel(216 * timeout_us, pcie->base + REG_OFFSET); } @@ -1676,6 +1680,13 @@ static const int pcie_offsets_bmips_7425[] = { [PCIE_INTR2_CPU_BASE] = 0x4300, }; +static const int pcie_offset_bcm7712[] = { + [EXT_CFG_INDEX] = 0x9000, + [EXT_CFG_DATA] = 0x9004, + [PCIE_HARD_DEBUG] = 0x4304, + [PCIE_INTR2_CPU_BASE] = 0x4400, +}; + static const struct pcie_cfg_data generic_cfg = { .offsets = pcie_offsets, .soc_base = GENERIC, @@ -1741,6 +1752,14 @@ static const struct pcie_cfg_data bcm7216_cfg = { .num_inbound_wins = 3, }; +static const struct pcie_cfg_data bcm7712_cfg = { + .offsets = pcie_offset_bcm7712, + .perst_set = brcm_pcie_perst_set_7278, + .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, + .soc_base = BCM7712, + .num_inbound_wins = 10, +}; + static const struct of_device_id brcm_pcie_match[] = { { .compatible = "brcm,bcm2711-pcie", .data = &bcm2711_cfg }, { .compatible = "brcm,bcm4908-pcie", .data = &bcm4908_cfg }, @@ -1750,6 +1769,7 @@ static const struct of_device_id brcm_pcie_match[] = { { .compatible = "brcm,bcm7445-pcie", .data = &generic_cfg }, { .compatible = "brcm,bcm7435-pcie", .data = &bcm7435_cfg }, { .compatible = "brcm,bcm7425-pcie", .data = &bcm7425_cfg }, + { .compatible = "brcm,bcm7712-pcie", .data = &bcm7712_cfg }, {}, }; From a3ed1cc4136df01e74a8b6df8c562db69e752bad Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Sat, 7 Sep 2024 18:00:26 +0200 Subject: [PATCH 120/302] bcachefs: Fix negative timespecs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This fixes two problems in the handling of negative times: • rem is signed, but the rem * c->sb.nsec_per_time_unit operation produced a bogus unsigned result, because s32 * u32 = u32. • The timespec was not normalized (it could contain more than a billion nanoseconds). For example, { .tv_sec = -14245441, .tv_nsec = 750000000 }, after being round tripped through timespec_to_bch2_time and then bch2_time_to_timespec would come back as { .tv_sec = -14245440, .tv_nsec = 4044967296 } (more than 4 billion nanoseconds). Cc: stable@vger.kernel.org Fixes: 595c1e9bab7f ("bcachefs: Fix time handling") Closes: https://github.com/koverstreet/bcachefs/issues/743 Co-developed-by: Erin Shepherd Signed-off-by: Erin Shepherd Co-developed-by: Ryan Lahfa Signed-off-by: Ryan Lahfa Signed-off-by: Alyssa Ross Signed-off-by: Kent Overstreet --- fs/bcachefs/bcachefs.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h index 0c7086e00d18..81c4d935cca8 100644 --- a/fs/bcachefs/bcachefs.h +++ b/fs/bcachefs/bcachefs.h @@ -1195,12 +1195,15 @@ static inline bool btree_id_cached(const struct bch_fs *c, enum btree_id btree) static inline struct timespec64 bch2_time_to_timespec(const struct bch_fs *c, s64 time) { struct timespec64 t; + s64 sec; s32 rem; time += c->sb.time_base_lo; - t.tv_sec = div_s64_rem(time, c->sb.time_units_per_sec, &rem); - t.tv_nsec = rem * c->sb.nsec_per_time_unit; + sec = div_s64_rem(time, c->sb.time_units_per_sec, &rem); + + set_normalized_timespec64(&t, sec, rem * (s64)c->sb.nsec_per_time_unit); + return t; } From 36f0af4f44a4e465b0af4bf188d780511c1ca189 Mon Sep 17 00:00:00 2001 From: Feiko Nanninga Date: Sun, 1 Sep 2024 19:08:05 +0200 Subject: [PATCH 121/302] bcachefs: Fix sysfs rebalance duration waited formatting cat /sys/fs/bcachefs/*/internal/rebalance_status waiting io wait duration: 13.5 GiB io wait remaining: 627 MiB duration waited: 1392 m duration waited was increasing at a rate of about 14 times the expected rate. div_u64 takes a u32 divisor, but u->nsecs (from time_units[]) can be bigger than u32. Signed-off-by: Feiko Nanninga Signed-off-by: Kent Overstreet --- fs/bcachefs/util.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/bcachefs/util.c b/fs/bcachefs/util.c index 1b8554460af4..a6c9c96955f1 100644 --- a/fs/bcachefs/util.c +++ b/fs/bcachefs/util.c @@ -360,7 +360,7 @@ void bch2_pr_time_units(struct printbuf *out, u64 ns) { const struct time_unit *u = bch2_pick_time_units(ns); - prt_printf(out, "%llu %s", div_u64(ns, u->nsecs), u->name); + prt_printf(out, "%llu %s", div64_u64(ns, u->nsecs), u->name); } static void bch2_pr_time_units_aligned(struct printbuf *out, u64 ns) From 27663d7784b5dfd354a968e06b26452dc93f2a16 Mon Sep 17 00:00:00 2001 From: Reed Riley Date: Thu, 5 Sep 2024 16:59:29 +0000 Subject: [PATCH 122/302] bcachefs: Replace div_u64 with div64_u64 where second param is u64 Bcachefs often uses this function to divide by nanosecond times - which can easily cause problems when cast to u32. For example, `cat /sys/fs/bcachefs/*/internal/rebalance_status` would return invalid data in the `duration waited` field because dividing by the number of nanoseconds in a minute requires the divisor parameter to be u64. Signed-off-by: Reed Riley Signed-off-by: Kent Overstreet --- fs/bcachefs/sysfs.c | 2 +- fs/bcachefs/util.c | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/fs/bcachefs/sysfs.c b/fs/bcachefs/sysfs.c index 33f2a64c14c9..abaf9510fd96 100644 --- a/fs/bcachefs/sysfs.c +++ b/fs/bcachefs/sysfs.c @@ -288,7 +288,7 @@ static int bch2_compression_stats_to_text(struct printbuf *out, struct bch_fs *c prt_tab_rjust(out); prt_human_readable_u64(out, nr_extents - ? div_u64(sectors_uncompressed << 9, nr_extents) + ? div64_u64(sectors_uncompressed << 9, nr_extents) : 0); prt_tab_rjust(out); prt_newline(out); diff --git a/fs/bcachefs/util.c b/fs/bcachefs/util.c index a6c9c96955f1..144d8df48b1e 100644 --- a/fs/bcachefs/util.c +++ b/fs/bcachefs/util.c @@ -64,7 +64,7 @@ static int bch2_pow(u64 n, u64 p, u64 *res) *res = 1; while (p--) { - if (*res > div_u64(U64_MAX, n)) + if (*res > div64_u64(U64_MAX, n)) return -ERANGE; *res *= n; } @@ -140,14 +140,14 @@ static int __bch2_strtou64_h(const char *cp, u64 *res) parse_or_ret(cp, parse_unit_suffix(cp, &b)); - if (v > div_u64(U64_MAX, b)) + if (v > div64_u64(U64_MAX, b)) return -ERANGE; v *= b; - if (f_n > div_u64(U64_MAX, b)) + if (f_n > div64_u64(U64_MAX, b)) return -ERANGE; - f_n = div_u64(f_n * b, f_d); + f_n = div64_u64(f_n * b, f_d); if (v + f_n < v) return -ERANGE; v += f_n; @@ -477,7 +477,7 @@ void bch2_time_stats_to_text(struct printbuf *out, struct bch2_time_stats *stats bool is_last = eytzinger0_next(i, NR_QUANTILES) == -1; u64 q = max(quantiles->entries[i].m, last_q); - prt_printf(out, "%llu ", div_u64(q, u->nsecs)); + prt_printf(out, "%llu ", div64_u64(q, u->nsecs)); if (is_last) prt_newline(out); last_q = q; From 88d2ae0e6eb84e1ed58f339c6a0de16c24fa2a60 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 8 Aug 2024 11:18:21 -0400 Subject: [PATCH 123/302] inode: make __iget() a static inline bcachefs is switching to an rhashtable for vfs inodes instead of the standard inode.c hashtable, so we need this exported, or - a static inline makes more sense for a single atomic_inc(). Signed-off-by: Kent Overstreet --- fs/inode.c | 8 -------- include/linux/fs.h | 9 ++++++++- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/fs/inode.c b/fs/inode.c index 86670941884b..5e7dcdeedd4d 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -438,14 +438,6 @@ static void init_once(void *foo) inode_init_once(inode); } -/* - * inode->i_lock must be held - */ -void __iget(struct inode *inode) -{ - atomic_inc(&inode->i_count); -} - /* * get additional reference to inode; caller must already hold one. */ diff --git a/include/linux/fs.h b/include/linux/fs.h index fd34b5755c0b..8fc4bad3b6ae 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -3094,7 +3094,14 @@ static inline bool is_zero_ino(ino_t ino) return (u32)ino == 0; } -extern void __iget(struct inode * inode); +/* + * inode->i_lock must be held + */ +static inline void __iget(struct inode *inode) +{ + atomic_inc(&inode->i_count); +} + extern void iget_failed(struct inode *); extern void clear_inode(struct inode *); extern void __destroy_inode(struct inode *); From 112d21fd1a122f778c383aa44d5448f4da9518c3 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sat, 8 Jun 2024 21:41:01 -0400 Subject: [PATCH 124/302] bcachefs: switch to rhashtable for vfs inodes hash the standard vfs inode hash table suffers from painful lock contention - this is long overdue Signed-off-by: Kent Overstreet --- fs/bcachefs/acl.c | 2 +- fs/bcachefs/bcachefs.h | 1 + fs/bcachefs/fs-io-buffered.c | 2 +- fs/bcachefs/fs-io-direct.c | 2 +- fs/bcachefs/fs-io.c | 6 +- fs/bcachefs/fs-ioctl.c | 4 +- fs/bcachefs/fs.c | 205 ++++++++++++++++++++++------------ fs/bcachefs/fs.h | 18 ++- fs/bcachefs/inode.c | 2 +- fs/bcachefs/subvolume_types.h | 3 +- fs/bcachefs/super.c | 2 + fs/bcachefs/xattr.c | 2 +- 12 files changed, 160 insertions(+), 89 deletions(-) diff --git a/fs/bcachefs/acl.c b/fs/bcachefs/acl.c index 331a17f3f113..87f1be9d4db4 100644 --- a/fs/bcachefs/acl.c +++ b/fs/bcachefs/acl.c @@ -361,7 +361,7 @@ retry: bch2_trans_begin(trans); acl = _acl; - ret = bch2_subvol_is_ro_trans(trans, inode->ei_subvol) ?: + ret = bch2_subvol_is_ro_trans(trans, inode->ei_inum.subvol) ?: bch2_inode_peek(trans, &inode_iter, &inode_u, inode_inum(inode), BTREE_ITER_intent); if (ret) diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h index 81c4d935cca8..ffc90615eba4 100644 --- a/fs/bcachefs/bcachefs.h +++ b/fs/bcachefs/bcachefs.h @@ -1023,6 +1023,7 @@ struct bch_fs { /* fs.c */ struct list_head vfs_inodes_list; struct mutex vfs_inodes_lock; + struct rhashtable vfs_inodes_table; /* VFS IO PATH - fs-io.c */ struct bio_set writepage_bioset; diff --git a/fs/bcachefs/fs-io-buffered.c b/fs/bcachefs/fs-io-buffered.c index ec8c427bf588..9fe1285908fa 100644 --- a/fs/bcachefs/fs-io-buffered.c +++ b/fs/bcachefs/fs-io-buffered.c @@ -486,7 +486,7 @@ static void bch2_writepage_io_alloc(struct bch_fs *c, op->nr_replicas = nr_replicas; op->res.nr_replicas = nr_replicas; op->write_point = writepoint_hashed(inode->ei_last_dirtied); - op->subvol = inode->ei_subvol; + op->subvol = inode->ei_inum.subvol; op->pos = POS(inode->v.i_ino, sector); op->end_io = bch2_writepage_io_done; op->devs_need_flush = &inode->ei_devs_need_flush; diff --git a/fs/bcachefs/fs-io-direct.c b/fs/bcachefs/fs-io-direct.c index e246b1e05aa2..ee1c0325f313 100644 --- a/fs/bcachefs/fs-io-direct.c +++ b/fs/bcachefs/fs-io-direct.c @@ -500,7 +500,7 @@ static __always_inline long bch2_dio_write_loop(struct dio_write *dio) dio->op.target = dio->op.opts.foreground_target; dio->op.write_point = writepoint_hashed((unsigned long) current); dio->op.nr_replicas = dio->op.opts.data_replicas; - dio->op.subvol = inode->ei_subvol; + dio->op.subvol = inode->ei_inum.subvol; dio->op.pos = POS(inode->v.i_ino, (u64) req->ki_pos >> 9); dio->op.devs_need_flush = &inode->ei_devs_need_flush; diff --git a/fs/bcachefs/fs-io.c b/fs/bcachefs/fs-io.c index 77b85da30fb2..a5018cbdb376 100644 --- a/fs/bcachefs/fs-io.c +++ b/fs/bcachefs/fs-io.c @@ -267,7 +267,7 @@ static int __bch2_truncate_folio(struct bch_inode_info *inode, * XXX: we're doing two index lookups when we end up reading the * folio */ - ret = range_has_data(c, inode->ei_subvol, + ret = range_has_data(c, inode->ei_inum.subvol, POS(inode->v.i_ino, (index << PAGE_SECTORS_SHIFT)), POS(inode->v.i_ino, (index << PAGE_SECTORS_SHIFT) + PAGE_SECTORS)); if (ret <= 0) @@ -618,7 +618,7 @@ static noinline int __bchfs_fallocate(struct bch_inode_info *inode, int mode, bch2_trans_begin(trans); ret = bch2_subvolume_get_snapshot(trans, - inode->ei_subvol, &snapshot); + inode->ei_inum.subvol, &snapshot); if (ret) goto bkey_err; @@ -823,7 +823,7 @@ static int quota_reserve_range(struct bch_inode_info *inode, retry: bch2_trans_begin(trans); - ret = bch2_subvolume_get_snapshot(trans, inode->ei_subvol, &snapshot); + ret = bch2_subvolume_get_snapshot(trans, inode->ei_inum.subvol, &snapshot); if (ret) goto err; diff --git a/fs/bcachefs/fs-ioctl.c b/fs/bcachefs/fs-ioctl.c index 99c7fe987c74..405cf08bda34 100644 --- a/fs/bcachefs/fs-ioctl.c +++ b/fs/bcachefs/fs-ioctl.c @@ -100,7 +100,7 @@ static int bch2_ioc_setflags(struct bch_fs *c, } mutex_lock(&inode->ei_update_lock); - ret = bch2_subvol_is_ro(c, inode->ei_subvol) ?: + ret = bch2_subvol_is_ro(c, inode->ei_inum.subvol) ?: bch2_write_inode(c, inode, bch2_inode_flags_set, &s, ATTR_CTIME); mutex_unlock(&inode->ei_update_lock); @@ -184,7 +184,7 @@ static int bch2_ioc_fssetxattr(struct bch_fs *c, } mutex_lock(&inode->ei_update_lock); - ret = bch2_subvol_is_ro(c, inode->ei_subvol) ?: + ret = bch2_subvol_is_ro(c, inode->ei_inum.subvol) ?: bch2_set_projid(c, inode, fa.fsx_projid) ?: bch2_write_inode(c, inode, fssetxattr_inode_update_fn, &s, ATTR_CTIME); diff --git a/fs/bcachefs/fs.c b/fs/bcachefs/fs.c index 257f07656e5f..fa88993c9e8b 100644 --- a/fs/bcachefs/fs.c +++ b/fs/bcachefs/fs.c @@ -108,7 +108,7 @@ retry: goto retry; bch2_fs_fatal_err_on(bch2_err_matches(ret, ENOENT), c, - "%s: inode %u:%llu not found when updating", + "%s: inode %llu:%llu not found when updating", bch2_err_str(ret), inode_inum(inode).subvol, inode_inum(inode).inum); @@ -152,50 +152,95 @@ int bch2_fs_quota_transfer(struct bch_fs *c, return ret; } -static int bch2_iget5_test(struct inode *vinode, void *p) +static bool subvol_inum_eq(subvol_inum a, subvol_inum b) { - struct bch_inode_info *inode = to_bch_ei(vinode); - subvol_inum *inum = p; - - return inode->ei_subvol == inum->subvol && - inode->ei_inode.bi_inum == inum->inum; + return a.subvol == b.subvol && a.inum == b.inum; } -static int bch2_iget5_set(struct inode *vinode, void *p) +static int bch2_vfs_inode_cmp_fn(struct rhashtable_compare_arg *arg, + const void *obj) { - struct bch_inode_info *inode = to_bch_ei(vinode); - subvol_inum *inum = p; + const struct bch_inode_info *inode = obj; + const subvol_inum *v = arg->key; - inode->v.i_ino = inum->inum; - inode->ei_subvol = inum->subvol; - inode->ei_inode.bi_inum = inum->inum; - return 0; + return !subvol_inum_eq(inode->ei_inum, *v); } -static unsigned bch2_inode_hash(subvol_inum inum) +static const struct rhashtable_params bch2_vfs_inodes_params = { + .head_offset = offsetof(struct bch_inode_info, hash), + .key_offset = offsetof(struct bch_inode_info, ei_inum), + .key_len = sizeof(subvol_inum), + .obj_cmpfn = bch2_vfs_inode_cmp_fn, + .automatic_shrinking = true, +}; + +static void __wait_on_freeing_inode(struct inode *inode) { - return jhash_3words(inum.subvol, inum.inum >> 32, inum.inum, JHASH_INITVAL); + wait_queue_head_t *wq; + DEFINE_WAIT_BIT(wait, &inode->i_state, __I_NEW); + wq = bit_waitqueue(&inode->i_state, __I_NEW); + prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE); + spin_unlock(&inode->i_lock); + schedule(); + finish_wait(wq, &wait.wq_entry); } struct bch_inode_info *__bch2_inode_hash_find(struct bch_fs *c, subvol_inum inum) { - return to_bch_ei(ilookup5_nowait(c->vfs_sb, - bch2_inode_hash(inum), - bch2_iget5_test, - &inum)); + return rhashtable_lookup_fast(&c->vfs_inodes_table, &inum, bch2_vfs_inodes_params); } -static struct bch_inode_info *bch2_inode_insert(struct bch_fs *c, struct bch_inode_info *inode) +static struct bch_inode_info *bch2_inode_hash_find(struct bch_fs *c, subvol_inum inum) { - subvol_inum inum = inode_inum(inode); - struct bch_inode_info *old = to_bch_ei(inode_insert5(&inode->v, - bch2_inode_hash(inum), - bch2_iget5_test, - bch2_iget5_set, - &inum)); - BUG_ON(!old); + struct bch_inode_info *inode; +repeat: + inode = __bch2_inode_hash_find(c, inum); + if (inode) { + spin_lock(&inode->v.i_lock); + if (!test_bit(EI_INODE_HASHED, &inode->ei_flags)) { + spin_unlock(&inode->v.i_lock); + return NULL; + } + if ((inode->v.i_state & (I_FREEING|I_WILL_FREE))) { + __wait_on_freeing_inode(&inode->v); + goto repeat; + } + __iget(&inode->v); + spin_unlock(&inode->v.i_lock); + } + + return inode; +} + +static void bch2_inode_hash_remove(struct bch_fs *c, struct bch_inode_info *inode) +{ + spin_lock(&inode->v.i_lock); + bool remove = test_and_clear_bit(EI_INODE_HASHED, &inode->ei_flags); + spin_unlock(&inode->v.i_lock); + + if (remove) { + int ret = rhashtable_remove_fast(&c->vfs_inodes_table, + &inode->hash, bch2_vfs_inodes_params); + BUG_ON(ret); + inode->v.i_hash.pprev = NULL; + } +} + +static struct bch_inode_info *bch2_inode_hash_insert(struct bch_fs *c, struct bch_inode_info *inode) +{ + struct bch_inode_info *old = inode; + + set_bit(EI_INODE_HASHED, &inode->ei_flags); +retry: + if (unlikely(rhashtable_lookup_insert_fast(&c->vfs_inodes_table, + &inode->hash, + bch2_vfs_inodes_params))) { + old = bch2_inode_hash_find(c, inode->ei_inum); + if (!old) + goto retry; + + clear_bit(EI_INODE_HASHED, &inode->ei_flags); - if (unlikely(old != inode)) { /* * bcachefs doesn't use I_NEW; we have no use for it since we * only insert fully created inodes in the inode hash table. But @@ -211,16 +256,13 @@ static struct bch_inode_info *bch2_inode_insert(struct bch_fs *c, struct bch_ino discard_new_inode(&inode->v); inode = old; } else { + inode_fake_hash(&inode->v); + + inode_sb_list_add(&inode->v); + mutex_lock(&c->vfs_inodes_lock); list_add(&inode->ei_vfs_inode_list, &c->vfs_inodes_list); mutex_unlock(&c->vfs_inodes_lock); - /* - * Again, I_NEW makes no sense for bcachefs. This is only needed - * for clearing I_NEW, but since the inode was already fully - * created and initialized we didn't actually want - * inode_insert5() to set it for us. - */ - unlock_new_inode(&inode->v); } return inode; @@ -285,11 +327,7 @@ static struct bch_inode_info *bch2_new_inode(struct btree_trans *trans) struct inode *bch2_vfs_inode_get(struct bch_fs *c, subvol_inum inum) { - struct bch_inode_info *inode = - to_bch_ei(ilookup5_nowait(c->vfs_sb, - bch2_inode_hash(inum), - bch2_iget5_test, - &inum)); + struct bch_inode_info *inode = bch2_inode_hash_find(c, inum); if (inode) return &inode->v; @@ -303,7 +341,7 @@ struct inode *bch2_vfs_inode_get(struct bch_fs *c, subvol_inum inum) PTR_ERR_OR_ZERO(inode = bch2_new_inode(trans)); if (!ret) { bch2_vfs_inode_init(trans, inum, inode, &inode_u, &subvol); - inode = bch2_inode_insert(c, inode); + inode = bch2_inode_hash_insert(c, inode); } bch2_trans_put(trans); @@ -351,7 +389,7 @@ __bch2_create(struct mnt_idmap *idmap, retry: bch2_trans_begin(trans); - ret = bch2_subvol_is_ro_trans(trans, dir->ei_subvol) ?: + ret = bch2_subvol_is_ro_trans(trans, dir->ei_inum.subvol) ?: bch2_create_trans(trans, inode_inum(dir), &dir_u, &inode_u, !(flags & BCH_CREATE_TMPFILE) @@ -365,7 +403,7 @@ retry: if (unlikely(ret)) goto err_before_quota; - inum.subvol = inode_u.bi_subvol ?: dir->ei_subvol; + inum.subvol = inode_u.bi_subvol ?: dir->ei_inum.subvol; inum.inum = inode_u.bi_inum; ret = bch2_subvolume_get(trans, inum.subvol, true, @@ -396,7 +434,7 @@ err_before_quota: * bch2_trans_exit() and dropping locks, else we could race with another * thread pulling the inode in and modifying it: */ - inode = bch2_inode_insert(c, inode); + inode = bch2_inode_hash_insert(c, inode); bch2_trans_put(trans); err: posix_acl_release(default_acl); @@ -436,11 +474,7 @@ static struct bch_inode_info *bch2_lookup_trans(struct btree_trans *trans, if (ret) goto err; - struct bch_inode_info *inode = - to_bch_ei(ilookup5_nowait(c->vfs_sb, - bch2_inode_hash(inum), - bch2_iget5_test, - &inum)); + struct bch_inode_info *inode = bch2_inode_hash_find(c, inum); if (inode) goto out; @@ -470,7 +504,7 @@ static struct bch_inode_info *bch2_lookup_trans(struct btree_trans *trans, } bch2_vfs_inode_init(trans, inum, inode, &inode_u, &subvol); - inode = bch2_inode_insert(c, inode); + inode = bch2_inode_hash_insert(c, inode); out: bch2_trans_iter_exit(trans, &dirent_iter); printbuf_exit(&buf); @@ -557,8 +591,8 @@ static int bch2_link(struct dentry *old_dentry, struct inode *vdir, lockdep_assert_held(&inode->v.i_rwsem); - ret = bch2_subvol_is_ro(c, dir->ei_subvol) ?: - bch2_subvol_is_ro(c, inode->ei_subvol) ?: + ret = bch2_subvol_is_ro(c, dir->ei_inum.subvol) ?: + bch2_subvol_is_ro(c, inode->ei_inum.subvol) ?: __bch2_link(c, inode, dir, dentry); if (unlikely(ret)) return bch2_err_class(ret); @@ -614,7 +648,7 @@ static int bch2_unlink(struct inode *vdir, struct dentry *dentry) struct bch_inode_info *dir= to_bch_ei(vdir); struct bch_fs *c = dir->v.i_sb->s_fs_info; - int ret = bch2_subvol_is_ro(c, dir->ei_subvol) ?: + int ret = bch2_subvol_is_ro(c, dir->ei_inum.subvol) ?: __bch2_unlink(vdir, dentry, false); return bch2_err_class(ret); } @@ -697,8 +731,8 @@ static int bch2_rename2(struct mnt_idmap *idmap, trans = bch2_trans_get(c); - ret = bch2_subvol_is_ro_trans(trans, src_dir->ei_subvol) ?: - bch2_subvol_is_ro_trans(trans, dst_dir->ei_subvol); + ret = bch2_subvol_is_ro_trans(trans, src_dir->ei_inum.subvol) ?: + bch2_subvol_is_ro_trans(trans, dst_dir->ei_inum.subvol); if (ret) goto err; @@ -899,7 +933,7 @@ static int bch2_getattr(struct mnt_idmap *idmap, stat->blksize = block_bytes(c); stat->blocks = inode->v.i_blocks; - stat->subvol = inode->ei_subvol; + stat->subvol = inode->ei_inum.subvol; stat->result_mask |= STATX_SUBVOL; if ((request_mask & STATX_DIOALIGN) && S_ISREG(inode->v.i_mode)) { @@ -941,7 +975,7 @@ static int bch2_setattr(struct mnt_idmap *idmap, lockdep_assert_held(&inode->v.i_rwsem); - ret = bch2_subvol_is_ro(c, inode->ei_subvol) ?: + ret = bch2_subvol_is_ro(c, inode->ei_inum.subvol) ?: setattr_prepare(idmap, dentry, iattr); if (ret) return ret; @@ -1053,7 +1087,7 @@ static int bch2_fiemap(struct inode *vinode, struct fiemap_extent_info *info, retry: bch2_trans_begin(trans); - ret = bch2_subvolume_get_snapshot(trans, ei->ei_subvol, &snapshot); + ret = bch2_subvolume_get_snapshot(trans, ei->ei_inum.subvol, &snapshot); if (ret) goto err; @@ -1173,7 +1207,7 @@ static int bch2_open(struct inode *vinode, struct file *file) struct bch_inode_info *inode = to_bch_ei(vinode); struct bch_fs *c = inode->v.i_sb->s_fs_info; - int ret = bch2_subvol_is_ro(c, inode->ei_subvol); + int ret = bch2_subvol_is_ro(c, inode->ei_inum.subvol); if (ret) return ret; } @@ -1305,8 +1339,8 @@ static int bcachefs_fid_valid(int fh_len, int fh_type) static struct bcachefs_fid bch2_inode_to_fid(struct bch_inode_info *inode) { return (struct bcachefs_fid) { - .inum = inode->ei_inode.bi_inum, - .subvol = inode->ei_subvol, + .inum = inode->ei_inum.inum, + .subvol = inode->ei_inum.subvol, .gen = inode->ei_inode.bi_generation, }; } @@ -1391,7 +1425,7 @@ static struct dentry *bch2_get_parent(struct dentry *child) struct bch_fs *c = inode->v.i_sb->s_fs_info; subvol_inum parent_inum = { .subvol = inode->ei_inode.bi_parent_subvol ?: - inode->ei_subvol, + inode->ei_inum.subvol, .inum = inode->ei_inode.bi_dir, }; @@ -1427,7 +1461,7 @@ static int bch2_get_name(struct dentry *parent, char *name, struct dentry *child retry: bch2_trans_begin(trans); - ret = bch2_subvolume_get_snapshot(trans, dir->ei_subvol, &snapshot); + ret = bch2_subvolume_get_snapshot(trans, dir->ei_inum.subvol, &snapshot); if (ret) goto err; @@ -1458,8 +1492,7 @@ retry: if (ret) goto err; - if (target.subvol == inode->ei_subvol && - target.inum == inode->ei_inode.bi_inum) + if (subvol_inum_eq(target, inode->ei_inum)) goto found; } else { /* @@ -1480,8 +1513,7 @@ retry: if (ret) continue; - if (target.subvol == inode->ei_subvol && - target.inum == inode->ei_inode.bi_inum) + if (subvol_inum_eq(target, inode->ei_inum)) goto found; } } @@ -1518,7 +1550,9 @@ static void bch2_vfs_inode_init(struct btree_trans *trans, subvol_inum inum, struct bch_inode_unpacked *bi, struct bch_subvolume *subvol) { - bch2_iget5_set(&inode->v, &inum); + inode->v.i_ino = inum.inum; + inode->ei_inum = inum; + inode->ei_inode.bi_inum = inum.inum; bch2_inode_update_after_write(trans, inode, bi, ~0); inode->v.i_blocks = bi->bi_sectors; @@ -1530,7 +1564,6 @@ static void bch2_vfs_inode_init(struct btree_trans *trans, subvol_inum inum, inode->ei_flags = 0; inode->ei_quota_reserved = 0; inode->ei_qid = bch_qid(bi); - inode->ei_subvol = inum.subvol; if (BCH_SUBVOLUME_SNAP(subvol)) set_bit(EI_INODE_SNAPSHOT, &inode->ei_flags); @@ -1597,6 +1630,17 @@ static void bch2_evict_inode(struct inode *vinode) { struct bch_fs *c = vinode->i_sb->s_fs_info; struct bch_inode_info *inode = to_bch_ei(vinode); + bool delete = !inode->v.i_nlink && !is_bad_inode(&inode->v); + + /* + * evict() has waited for outstanding writeback, we'll do no more IO + * through this inode: it's safe to remove from VFS inode hashtable here + * + * Do that now so that other threads aren't blocked from pulling it back + * in, there's no reason for them to be: + */ + if (!delete) + bch2_inode_hash_remove(c, inode); truncate_inode_pages_final(&inode->v.i_data); @@ -1604,12 +1648,18 @@ static void bch2_evict_inode(struct inode *vinode) BUG_ON(!is_bad_inode(&inode->v) && inode->ei_quota_reserved); - if (!inode->v.i_nlink && !is_bad_inode(&inode->v)) { + if (delete) { bch2_quota_acct(c, inode->ei_qid, Q_SPC, -((s64) inode->v.i_blocks), KEY_TYPE_QUOTA_WARN); bch2_quota_acct(c, inode->ei_qid, Q_INO, -1, KEY_TYPE_QUOTA_WARN); bch2_inode_rm(c, inode_inum(inode)); + + /* + * If we are deleting, we need it present in the vfs hash table + * so that fsck can check if unlinked inodes are still open: + */ + bch2_inode_hash_remove(c, inode); } mutex_lock(&c->vfs_inodes_lock); @@ -1639,7 +1689,7 @@ again: mutex_lock(&c->vfs_inodes_lock); list_for_each_entry(inode, &c->vfs_inodes_list, ei_vfs_inode_list) { - if (!snapshot_list_has_id(s, inode->ei_subvol)) + if (!snapshot_list_has_id(s, inode->ei_inum.subvol)) continue; if (!(inode->v.i_state & I_DONTCACHE) && @@ -2127,6 +2177,17 @@ static int bch2_init_fs_context(struct fs_context *fc) return 0; } +void bch2_fs_vfs_exit(struct bch_fs *c) +{ + if (c->vfs_inodes_table.tbl) + rhashtable_destroy(&c->vfs_inodes_table); +} + +int bch2_fs_vfs_init(struct bch_fs *c) +{ + return rhashtable_init(&c->vfs_inodes_table, &bch2_vfs_inodes_params); +} + static struct file_system_type bcache_fs_type = { .owner = THIS_MODULE, .name = "bcachefs", diff --git a/fs/bcachefs/fs.h b/fs/bcachefs/fs.h index 990ec43e0365..da74ecc236e7 100644 --- a/fs/bcachefs/fs.h +++ b/fs/bcachefs/fs.h @@ -13,6 +13,9 @@ struct bch_inode_info { struct inode v; + struct rhash_head hash; + subvol_inum ei_inum; + struct list_head ei_vfs_inode_list; unsigned long ei_flags; @@ -24,8 +27,6 @@ struct bch_inode_info { struct mutex ei_quota_lock; struct bch_qid ei_qid; - u32 ei_subvol; - /* * When we've been doing nocow writes we'll need to issue flushes to the * underlying block devices @@ -50,10 +51,7 @@ struct bch_inode_info { static inline subvol_inum inode_inum(struct bch_inode_info *inode) { - return (subvol_inum) { - .subvol = inode->ei_subvol, - .inum = inode->ei_inode.bi_inum, - }; + return inode->ei_inum; } struct bch_inode_info *__bch2_inode_hash_find(struct bch_fs *, subvol_inum); @@ -69,6 +67,7 @@ struct bch_inode_info *__bch2_inode_hash_find(struct bch_fs *, subvol_inum); * those: */ #define EI_INODE_SNAPSHOT 1 +#define EI_INODE_HASHED 2 #define to_bch_ei(_inode) \ container_of_or_null(_inode, struct bch_inode_info, v) @@ -189,6 +188,9 @@ int __bch2_unlink(struct inode *, struct dentry *, bool); void bch2_evict_subvolume_inodes(struct bch_fs *, snapshot_id_list *); +void bch2_fs_vfs_exit(struct bch_fs *); +int bch2_fs_vfs_init(struct bch_fs *); + void bch2_vfs_exit(void); int bch2_vfs_init(void); @@ -203,6 +205,10 @@ static inline struct bch_inode_info *__bch2_inode_hash_find(struct bch_fs *c, su static inline void bch2_evict_subvolume_inodes(struct bch_fs *c, snapshot_id_list *s) {} + +static inline void bch2_fs_vfs_exit(struct bch_fs *c) {} +static inline int bch2_fs_vfs_init(struct bch_fs *c) { return 0; } + static inline void bch2_vfs_exit(void) {} static inline int bch2_vfs_init(void) { return 0; } diff --git a/fs/bcachefs/inode.c b/fs/bcachefs/inode.c index 2be6be33afa3..6ac0ff7e074b 100644 --- a/fs/bcachefs/inode.c +++ b/fs/bcachefs/inode.c @@ -365,7 +365,7 @@ int bch2_inode_peek(struct btree_trans *trans, subvol_inum inum, unsigned flags) { int ret = bch2_inode_peek_nowarn(trans, iter, inode, inum, flags); - bch_err_msg(trans->c, ret, "looking up inum %u:%llu:", inum.subvol, inum.inum); + bch_err_msg(trans->c, ret, "looking up inum %llu:%llu:", inum.subvol, inum.inum); return ret; } diff --git a/fs/bcachefs/subvolume_types.h b/fs/bcachefs/subvolume_types.h index 9b10c8947828..f2ec4277c2a5 100644 --- a/fs/bcachefs/subvolume_types.h +++ b/fs/bcachefs/subvolume_types.h @@ -30,7 +30,8 @@ struct snapshot_table { }; typedef struct { - u32 subvol; + /* we can't have padding in this struct: */ + u64 subvol; u64 inum; } subvol_inum; diff --git a/fs/bcachefs/super.c b/fs/bcachefs/super.c index e7fa2de35014..6d0ce3d73450 100644 --- a/fs/bcachefs/super.c +++ b/fs/bcachefs/super.c @@ -543,6 +543,7 @@ static void __bch2_fs_free(struct bch_fs *c) bch2_fs_fs_io_direct_exit(c); bch2_fs_fs_io_buffered_exit(c); bch2_fs_fsio_exit(c); + bch2_fs_vfs_exit(c); bch2_fs_ec_exit(c); bch2_fs_encryption_exit(c); bch2_fs_nocow_locking_exit(c); @@ -926,6 +927,7 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts) bch2_fs_encryption_init(c) ?: bch2_fs_compress_init(c) ?: bch2_fs_ec_init(c) ?: + bch2_fs_vfs_init(c) ?: bch2_fs_fsio_init(c) ?: bch2_fs_fs_io_buffered_init(c) ?: bch2_fs_fs_io_direct_init(c); diff --git a/fs/bcachefs/xattr.c b/fs/bcachefs/xattr.c index 331f944d73dc..4b5898d3eda0 100644 --- a/fs/bcachefs/xattr.c +++ b/fs/bcachefs/xattr.c @@ -306,7 +306,7 @@ retry: bch2_trans_begin(trans); iter = (struct btree_iter) { NULL }; - ret = bch2_subvolume_get_snapshot(trans, inode->ei_subvol, &snapshot); + ret = bch2_subvolume_get_snapshot(trans, inode->ei_inum.subvol, &snapshot); if (ret) goto err; From 54f7702466b3a10b9a81e3c1c2c9da33df7a655f Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Fri, 16 Aug 2024 12:31:53 -0400 Subject: [PATCH 125/302] bcachefs: Fix deadlock in __wait_on_freeing_inode() We can't call __wait_on_freeing_inode() with btree locks held; we're waiting on another thread that's in evict(), and before it clears that bit it needs to write that inode to flush timestamps - deadlock. Fixing this involves a fair amount of re-jiggering to plumb a new transaction restart. Signed-off-by: Kent Overstreet --- fs/bcachefs/fs.c | 67 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 20 deletions(-) diff --git a/fs/bcachefs/fs.c b/fs/bcachefs/fs.c index fa88993c9e8b..2505daaec458 100644 --- a/fs/bcachefs/fs.c +++ b/fs/bcachefs/fs.c @@ -190,7 +190,8 @@ struct bch_inode_info *__bch2_inode_hash_find(struct bch_fs *c, subvol_inum inum return rhashtable_lookup_fast(&c->vfs_inodes_table, &inum, bch2_vfs_inodes_params); } -static struct bch_inode_info *bch2_inode_hash_find(struct bch_fs *c, subvol_inum inum) +static struct bch_inode_info *bch2_inode_hash_find(struct bch_fs *c, struct btree_trans *trans, + subvol_inum inum) { struct bch_inode_info *inode; repeat: @@ -202,7 +203,15 @@ repeat: return NULL; } if ((inode->v.i_state & (I_FREEING|I_WILL_FREE))) { - __wait_on_freeing_inode(&inode->v); + if (!trans) { + __wait_on_freeing_inode(&inode->v); + } else { + bch2_trans_unlock(trans); + __wait_on_freeing_inode(&inode->v); + int ret = bch2_trans_relock(trans); + if (ret) + return ERR_PTR(ret); + } goto repeat; } __iget(&inode->v); @@ -226,7 +235,9 @@ static void bch2_inode_hash_remove(struct bch_fs *c, struct bch_inode_info *inod } } -static struct bch_inode_info *bch2_inode_hash_insert(struct bch_fs *c, struct bch_inode_info *inode) +static struct bch_inode_info *bch2_inode_hash_insert(struct bch_fs *c, + struct btree_trans *trans, + struct bch_inode_info *inode) { struct bch_inode_info *old = inode; @@ -235,7 +246,7 @@ retry: if (unlikely(rhashtable_lookup_insert_fast(&c->vfs_inodes_table, &inode->hash, bch2_vfs_inodes_params))) { - old = bch2_inode_hash_find(c, inode->ei_inum); + old = bch2_inode_hash_find(c, trans, inode->ei_inum); if (!old) goto retry; @@ -254,7 +265,7 @@ retry: */ set_nlink(&inode->v, 1); discard_new_inode(&inode->v); - inode = old; + return old; } else { inode_fake_hash(&inode->v); @@ -263,9 +274,8 @@ retry: mutex_lock(&c->vfs_inodes_lock); list_add(&inode->ei_vfs_inode_list, &c->vfs_inodes_list); mutex_unlock(&c->vfs_inodes_lock); + return inode; } - - return inode; } #define memalloc_flags_do(_flags, _do) \ @@ -325,9 +335,24 @@ static struct bch_inode_info *bch2_new_inode(struct btree_trans *trans) return inode; } +static struct bch_inode_info *bch2_inode_hash_init_insert(struct btree_trans *trans, + subvol_inum inum, + struct bch_inode_unpacked *bi, + struct bch_subvolume *subvol) +{ + struct bch_inode_info *inode = bch2_new_inode(trans); + if (IS_ERR(inode)) + return inode; + + bch2_vfs_inode_init(trans, inum, inode, bi, subvol); + + return bch2_inode_hash_insert(trans->c, trans, inode); + +} + struct inode *bch2_vfs_inode_get(struct bch_fs *c, subvol_inum inum) { - struct bch_inode_info *inode = bch2_inode_hash_find(c, inum); + struct bch_inode_info *inode = bch2_inode_hash_find(c, NULL, inum); if (inode) return &inode->v; @@ -338,11 +363,7 @@ struct inode *bch2_vfs_inode_get(struct bch_fs *c, subvol_inum inum) int ret = lockrestart_do(trans, bch2_subvolume_get(trans, inum.subvol, true, 0, &subvol) ?: bch2_inode_find_by_inum_trans(trans, inum, &inode_u)) ?: - PTR_ERR_OR_ZERO(inode = bch2_new_inode(trans)); - if (!ret) { - bch2_vfs_inode_init(trans, inum, inode, &inode_u, &subvol); - inode = bch2_inode_hash_insert(c, inode); - } + PTR_ERR_OR_ZERO(inode = bch2_inode_hash_init_insert(trans, inum, &inode_u, &subvol)); bch2_trans_put(trans); return ret ? ERR_PTR(ret) : &inode->v; @@ -433,8 +454,16 @@ err_before_quota: * we must insert the new inode into the inode cache before calling * bch2_trans_exit() and dropping locks, else we could race with another * thread pulling the inode in and modifying it: + * + * also, calling bch2_inode_hash_insert() without passing in the + * transaction object is sketchy - if we could ever end up in + * __wait_on_freeing_inode(), we'd risk deadlock. + * + * But that shouldn't be possible, since we still have the inode locked + * that we just created, and we _really_ can't take a transaction + * restart here. */ - inode = bch2_inode_hash_insert(c, inode); + inode = bch2_inode_hash_insert(c, NULL, inode); bch2_trans_put(trans); err: posix_acl_release(default_acl); @@ -474,7 +503,7 @@ static struct bch_inode_info *bch2_lookup_trans(struct btree_trans *trans, if (ret) goto err; - struct bch_inode_info *inode = bch2_inode_hash_find(c, inum); + struct bch_inode_info *inode = bch2_inode_hash_find(c, trans, inum); if (inode) goto out; @@ -482,7 +511,7 @@ static struct bch_inode_info *bch2_lookup_trans(struct btree_trans *trans, struct bch_inode_unpacked inode_u; ret = bch2_subvolume_get(trans, inum.subvol, true, 0, &subvol) ?: bch2_inode_find_by_inum_nowarn_trans(trans, inum, &inode_u) ?: - PTR_ERR_OR_ZERO(inode = bch2_new_inode(trans)); + PTR_ERR_OR_ZERO(inode = bch2_inode_hash_init_insert(trans, inum, &inode_u, &subvol)); bch2_fs_inconsistent_on(bch2_err_matches(ret, ENOENT), c, "dirent to missing inode:\n %s", @@ -502,9 +531,6 @@ static struct bch_inode_info *bch2_lookup_trans(struct btree_trans *trans, ret = -ENOENT; goto err; } - - bch2_vfs_inode_init(trans, inum, inode, &inode_u, &subvol); - inode = bch2_inode_hash_insert(c, inode); out: bch2_trans_iter_exit(trans, &dirent_iter); printbuf_exit(&buf); @@ -1545,7 +1571,8 @@ static const struct export_operations bch_export_ops = { .get_name = bch2_get_name, }; -static void bch2_vfs_inode_init(struct btree_trans *trans, subvol_inum inum, +static void bch2_vfs_inode_init(struct btree_trans *trans, + subvol_inum inum, struct bch_inode_info *inode, struct bch_inode_unpacked *bi, struct bch_subvolume *subvol) From f6594633817374a64a5fb31007bd810cad1425ff Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Mon, 17 Jun 2024 19:00:33 -0400 Subject: [PATCH 126/302] lib/generic-radix-tree.c: genradix_ptr_inlined() Provide an inlined fast path Signed-off-by: Kent Overstreet --- include/linux/generic-radix-tree.h | 75 ++++++++++++++++++++++++++++++ lib/generic-radix-tree.c | 64 +------------------------ 2 files changed, 76 insertions(+), 63 deletions(-) diff --git a/include/linux/generic-radix-tree.h b/include/linux/generic-radix-tree.h index f3512fddf3d7..8a3e1e886d1c 100644 --- a/include/linux/generic-radix-tree.h +++ b/include/linux/generic-radix-tree.h @@ -48,6 +48,49 @@ struct genradix_root; #define GENRADIX_NODE_SHIFT 9 #define GENRADIX_NODE_SIZE (1U << GENRADIX_NODE_SHIFT) +#define GENRADIX_ARY (GENRADIX_NODE_SIZE / sizeof(struct genradix_node *)) +#define GENRADIX_ARY_SHIFT ilog2(GENRADIX_ARY) + +/* depth that's needed for a genradix that can address up to ULONG_MAX: */ +#define GENRADIX_MAX_DEPTH \ + DIV_ROUND_UP(BITS_PER_LONG - GENRADIX_NODE_SHIFT, GENRADIX_ARY_SHIFT) + +#define GENRADIX_DEPTH_MASK \ + ((unsigned long) (roundup_pow_of_two(GENRADIX_MAX_DEPTH + 1) - 1)) + +static inline int genradix_depth_shift(unsigned depth) +{ + return GENRADIX_NODE_SHIFT + GENRADIX_ARY_SHIFT * depth; +} + +/* + * Returns size (of data, in bytes) that a tree of a given depth holds: + */ +static inline size_t genradix_depth_size(unsigned depth) +{ + return 1UL << genradix_depth_shift(depth); +} + +static inline unsigned genradix_root_to_depth(struct genradix_root *r) +{ + return (unsigned long) r & GENRADIX_DEPTH_MASK; +} + +static inline struct genradix_node *genradix_root_to_node(struct genradix_root *r) +{ + return (void *) ((unsigned long) r & ~GENRADIX_DEPTH_MASK); +} + +struct genradix_node { + union { + /* Interior node: */ + struct genradix_node *children[GENRADIX_ARY]; + + /* Leaf: */ + u8 data[GENRADIX_NODE_SIZE]; + }; +}; + struct __genradix { struct genradix_root *root; }; @@ -128,6 +171,30 @@ static inline size_t __idx_to_offset(size_t idx, size_t obj_size) #define __genradix_idx_to_offset(_radix, _idx) \ __idx_to_offset(_idx, __genradix_obj_size(_radix)) +static inline void *__genradix_ptr_inlined(struct __genradix *radix, size_t offset) +{ + struct genradix_root *r = READ_ONCE(radix->root); + struct genradix_node *n = genradix_root_to_node(r); + unsigned level = genradix_root_to_depth(r); + unsigned shift = genradix_depth_shift(level); + + if (unlikely(ilog2(offset) >= genradix_depth_shift(level))) + return NULL; + + while (n && shift > GENRADIX_NODE_SHIFT) { + shift -= GENRADIX_ARY_SHIFT; + n = n->children[offset >> shift]; + offset &= (1UL << shift) - 1; + } + + return n ? &n->data[offset] : NULL; +} + +#define genradix_ptr_inlined(_radix, _idx) \ + (__genradix_cast(_radix) \ + __genradix_ptr_inlined(&(_radix)->tree, \ + __genradix_idx_to_offset(_radix, _idx))) + void *__genradix_ptr(struct __genradix *, size_t); /** @@ -144,6 +211,14 @@ void *__genradix_ptr(struct __genradix *, size_t); void *__genradix_ptr_alloc(struct __genradix *, size_t, gfp_t); +#define genradix_ptr_alloc_inlined(_radix, _idx, _gfp) \ + (__genradix_cast(_radix) \ + (__genradix_ptr_inlined(&(_radix)->tree, \ + __genradix_idx_to_offset(_radix, _idx)) ?: \ + __genradix_ptr_alloc(&(_radix)->tree, \ + __genradix_idx_to_offset(_radix, _idx), \ + _gfp))) + /** * genradix_ptr_alloc - get a pointer to a genradix entry, allocating it * if necessary diff --git a/lib/generic-radix-tree.c b/lib/generic-radix-tree.c index fa692c86f069..4efae0663049 100644 --- a/lib/generic-radix-tree.c +++ b/lib/generic-radix-tree.c @@ -5,75 +5,13 @@ #include #include -#define GENRADIX_ARY (GENRADIX_NODE_SIZE / sizeof(struct genradix_node *)) -#define GENRADIX_ARY_SHIFT ilog2(GENRADIX_ARY) - -struct genradix_node { - union { - /* Interior node: */ - struct genradix_node *children[GENRADIX_ARY]; - - /* Leaf: */ - u8 data[GENRADIX_NODE_SIZE]; - }; -}; - -static inline int genradix_depth_shift(unsigned depth) -{ - return GENRADIX_NODE_SHIFT + GENRADIX_ARY_SHIFT * depth; -} - -/* - * Returns size (of data, in bytes) that a tree of a given depth holds: - */ -static inline size_t genradix_depth_size(unsigned depth) -{ - return 1UL << genradix_depth_shift(depth); -} - -/* depth that's needed for a genradix that can address up to ULONG_MAX: */ -#define GENRADIX_MAX_DEPTH \ - DIV_ROUND_UP(BITS_PER_LONG - GENRADIX_NODE_SHIFT, GENRADIX_ARY_SHIFT) - -#define GENRADIX_DEPTH_MASK \ - ((unsigned long) (roundup_pow_of_two(GENRADIX_MAX_DEPTH + 1) - 1)) - -static inline unsigned genradix_root_to_depth(struct genradix_root *r) -{ - return (unsigned long) r & GENRADIX_DEPTH_MASK; -} - -static inline struct genradix_node *genradix_root_to_node(struct genradix_root *r) -{ - return (void *) ((unsigned long) r & ~GENRADIX_DEPTH_MASK); -} - /* * Returns pointer to the specified byte @offset within @radix, or NULL if not * allocated */ void *__genradix_ptr(struct __genradix *radix, size_t offset) { - struct genradix_root *r = READ_ONCE(radix->root); - struct genradix_node *n = genradix_root_to_node(r); - unsigned level = genradix_root_to_depth(r); - - if (ilog2(offset) >= genradix_depth_shift(level)) - return NULL; - - while (1) { - if (!n) - return NULL; - if (!level) - break; - - level--; - - n = n->children[offset >> genradix_depth_shift(level)]; - offset &= genradix_depth_size(level) - 1; - } - - return &n->data[offset]; + return __genradix_ptr_inlined(radix, offset); } EXPORT_SYMBOL(__genradix_ptr); From b3f9da79e7783ca4f1c1d4214af976c548629c1d Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sat, 10 Aug 2024 23:14:44 -0400 Subject: [PATCH 127/302] lib/generic-radix-tree.c: add preallocation Signed-off-by: Kent Overstreet --- include/linux/generic-radix-tree.h | 38 +++++++++++++++++++++++++----- lib/generic-radix-tree.c | 16 ++++--------- 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/include/linux/generic-radix-tree.h b/include/linux/generic-radix-tree.h index 8a3e1e886d1c..5b51c3d582d6 100644 --- a/include/linux/generic-radix-tree.h +++ b/include/linux/generic-radix-tree.h @@ -41,6 +41,7 @@ #include #include #include +#include #include struct genradix_root; @@ -81,6 +82,10 @@ static inline struct genradix_node *genradix_root_to_node(struct genradix_root * return (void *) ((unsigned long) r & ~GENRADIX_DEPTH_MASK); } +struct __genradix { + struct genradix_root *root; +}; + struct genradix_node { union { /* Interior node: */ @@ -91,9 +96,15 @@ struct genradix_node { }; }; -struct __genradix { - struct genradix_root *root; -}; +static inline struct genradix_node *genradix_alloc_node(gfp_t gfp_mask) +{ + return kzalloc(GENRADIX_NODE_SIZE, gfp_mask); +} + +static inline void genradix_free_node(struct genradix_node *node) +{ + kfree(node); +} /* * NOTE: currently, sizeof(_type) must not be larger than GENRADIX_NODE_SIZE: @@ -209,7 +220,8 @@ void *__genradix_ptr(struct __genradix *, size_t); __genradix_ptr(&(_radix)->tree, \ __genradix_idx_to_offset(_radix, _idx))) -void *__genradix_ptr_alloc(struct __genradix *, size_t, gfp_t); +void *__genradix_ptr_alloc(struct __genradix *, size_t, + struct genradix_node **, gfp_t); #define genradix_ptr_alloc_inlined(_radix, _idx, _gfp) \ (__genradix_cast(_radix) \ @@ -217,7 +229,15 @@ void *__genradix_ptr_alloc(struct __genradix *, size_t, gfp_t); __genradix_idx_to_offset(_radix, _idx)) ?: \ __genradix_ptr_alloc(&(_radix)->tree, \ __genradix_idx_to_offset(_radix, _idx), \ - _gfp))) + NULL, _gfp))) + +#define genradix_ptr_alloc_preallocated_inlined(_radix, _idx, _new_node, _gfp)\ + (__genradix_cast(_radix) \ + (__genradix_ptr_inlined(&(_radix)->tree, \ + __genradix_idx_to_offset(_radix, _idx)) ?: \ + __genradix_ptr_alloc(&(_radix)->tree, \ + __genradix_idx_to_offset(_radix, _idx), \ + _new_node, _gfp))) /** * genradix_ptr_alloc - get a pointer to a genradix entry, allocating it @@ -232,7 +252,13 @@ void *__genradix_ptr_alloc(struct __genradix *, size_t, gfp_t); (__genradix_cast(_radix) \ __genradix_ptr_alloc(&(_radix)->tree, \ __genradix_idx_to_offset(_radix, _idx), \ - _gfp)) + NULL, _gfp)) + +#define genradix_ptr_alloc_preallocated(_radix, _idx, _new_node, _gfp)\ + (__genradix_cast(_radix) \ + __genradix_ptr_alloc(&(_radix)->tree, \ + __genradix_idx_to_offset(_radix, _idx), \ + _new_node, _gfp)) struct genradix_iter { size_t offset; diff --git a/lib/generic-radix-tree.c b/lib/generic-radix-tree.c index 4efae0663049..79e067b51488 100644 --- a/lib/generic-radix-tree.c +++ b/lib/generic-radix-tree.c @@ -15,27 +15,21 @@ void *__genradix_ptr(struct __genradix *radix, size_t offset) } EXPORT_SYMBOL(__genradix_ptr); -static inline struct genradix_node *genradix_alloc_node(gfp_t gfp_mask) -{ - return kzalloc(GENRADIX_NODE_SIZE, gfp_mask); -} - -static inline void genradix_free_node(struct genradix_node *node) -{ - kfree(node); -} - /* * Returns pointer to the specified byte @offset within @radix, allocating it if * necessary - newly allocated slots are always zeroed out: */ void *__genradix_ptr_alloc(struct __genradix *radix, size_t offset, + struct genradix_node **preallocated, gfp_t gfp_mask) { struct genradix_root *v = READ_ONCE(radix->root); struct genradix_node *n, *new_node = NULL; unsigned level; + if (preallocated) + swap(new_node, *preallocated); + /* Increase tree depth if necessary: */ while (1) { struct genradix_root *r = v, *new_root; @@ -219,7 +213,7 @@ int __genradix_prealloc(struct __genradix *radix, size_t size, size_t offset; for (offset = 0; offset < size; offset += GENRADIX_NODE_SIZE) - if (!__genradix_ptr_alloc(radix, offset, gfp_mask)) + if (!__genradix_ptr_alloc(radix, offset, NULL, gfp_mask)) return -ENOMEM; return 0; From 8e973a4f3c74824ef03ed06006726321bc2346d6 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Mon, 10 Jun 2024 20:47:03 -0400 Subject: [PATCH 128/302] bcachefs: rcu_pending Generic data structure for explicitly tracking pending RCU items, allowing items to be dequeued (i.e. allocate from items pending freeing). Works with conventional RCU and SRCU, and possibly other RCU flavors in the future, meaning this can serve as a more generic replacement for SLAB_TYPESAFE_BY_RCU. Pending items are tracked in radix trees; if memory allocation fails, we fall back to linked lists. A rcu_pending is initialized with a callback, which is invoked when pending items's grace periods have expired. Two types of callback processing are handled specially: - RCU_PENDING_KVFREE_FN New backend for kvfree_rcu(). Slightly faster, and eliminates the synchronize_rcu() slowpath in kvfree_rcu_mightsleep() - instead, an rcu_head is allocated if we don't have one and can't use the radix tree TODO: - add a shrinker (as in the existing kvfree_rcu implementation) so that memory reclaim can free expired objects if callback processing isn't keeping up, and to expedite a grace period if we're under memory pressure and too much memory is stranded by RCU - add a counter for amount of memory pending - RCU_PENDING_CALL_RCU_FN Accelerated backend for call_rcu() - pending callbacks are tracked in a radix tree to eliminate linked list overhead. to serve as replacement backends for kvfree_rcu() and call_rcu(); these may be of interest to other uses (e.g. SLAB_TYPESAFE_BY_RCU users). Note: Internally, we're using a single rearming call_rcu() callback for notifications from the core RCU subsystem for notifications when objects are ready to be processed. Ideally we would be getting a callback every time a grace period completes for which we have objects, but that would require multiple rcu_heads in flight, and since the number of gp sequence numbers with uncompleted callbacks is not bounded, we can't do that yet. Signed-off-by: Kent Overstreet --- fs/bcachefs/Makefile | 1 + fs/bcachefs/rcu_pending.c | 603 ++++++++++++++++++++++++++++++++++++++ fs/bcachefs/rcu_pending.h | 25 ++ 3 files changed, 629 insertions(+) create mode 100644 fs/bcachefs/rcu_pending.c create mode 100644 fs/bcachefs/rcu_pending.h diff --git a/fs/bcachefs/Makefile b/fs/bcachefs/Makefile index 0ab533a2b03b..56d20e219f59 100644 --- a/fs/bcachefs/Makefile +++ b/fs/bcachefs/Makefile @@ -69,6 +69,7 @@ bcachefs-y := \ printbuf.o \ quota.o \ rebalance.o \ + rcu_pending.o \ recovery.o \ recovery_passes.o \ reflink.o \ diff --git a/fs/bcachefs/rcu_pending.c b/fs/bcachefs/rcu_pending.c new file mode 100644 index 000000000000..8f8d914d3998 --- /dev/null +++ b/fs/bcachefs/rcu_pending.c @@ -0,0 +1,603 @@ +// SPDX-License-Identifier: GPL-2.0 +#define pr_fmt(fmt) "%s() " fmt "\n", __func__ + +#include +#include +#include +#include +#include +#include + +#include "rcu_pending.h" +#include "darray.h" +#include "util.h" + +#define static_array_for_each(_a, _i) \ + for (typeof(&(_a)[0]) _i = _a; \ + _i < (_a) + ARRAY_SIZE(_a); \ + _i++) + +enum rcu_pending_special { + RCU_PENDING_KVFREE = 1, + RCU_PENDING_CALL_RCU = 2, +}; + +#define RCU_PENDING_KVFREE_FN ((rcu_pending_process_fn) (ulong) RCU_PENDING_KVFREE) +#define RCU_PENDING_CALL_RCU_FN ((rcu_pending_process_fn) (ulong) RCU_PENDING_CALL_RCU) + +static inline unsigned long __get_state_synchronize_rcu(struct srcu_struct *ssp) +{ + return ssp + ? get_state_synchronize_srcu(ssp) + : get_state_synchronize_rcu(); +} + +static inline unsigned long __start_poll_synchronize_rcu(struct srcu_struct *ssp) +{ + return ssp + ? start_poll_synchronize_srcu(ssp) + : start_poll_synchronize_rcu(); +} + +static inline bool __poll_state_synchronize_rcu(struct srcu_struct *ssp, unsigned long cookie) +{ + return ssp + ? poll_state_synchronize_srcu(ssp, cookie) + : poll_state_synchronize_rcu(cookie); +} + +static inline void __rcu_barrier(struct srcu_struct *ssp) +{ + return ssp + ? srcu_barrier(ssp) + : rcu_barrier(); +} + +static inline void __call_rcu(struct srcu_struct *ssp, struct rcu_head *rhp, + rcu_callback_t func) +{ + if (ssp) + call_srcu(ssp, rhp, func); + else + call_rcu(rhp, func); +} + +struct rcu_pending_seq { + /* + * We're using a radix tree like a vector - we're just pushing elements + * onto the end; we're using a radix tree instead of an actual vector to + * avoid reallocation overhead + */ + GENRADIX(struct rcu_head *) objs; + size_t nr; + struct rcu_head **cursor; + unsigned long seq; +}; + +struct rcu_pending_list { + struct rcu_head *head; + struct rcu_head *tail; + unsigned long seq; +}; + +struct rcu_pending_pcpu { + struct rcu_pending *parent; + spinlock_t lock; + int cpu; + + /* + * We can't bound the number of unprocessed gp sequence numbers, and we + * can't efficiently merge radix trees for expired grace periods, so we + * need darray/vector: + */ + DARRAY_PREALLOCATED(struct rcu_pending_seq, 4) objs; + + /* Third entry is for expired objects: */ + struct rcu_pending_list lists[NUM_ACTIVE_RCU_POLL_OLDSTATE + 1]; + + struct rcu_head cb; + bool cb_armed; + struct work_struct work; +}; + +static bool __rcu_pending_has_pending(struct rcu_pending_pcpu *p) +{ + if (p->objs.nr) + return true; + + static_array_for_each(p->lists, i) + if (i->head) + return true; + + return false; +} + +static void rcu_pending_list_merge(struct rcu_pending_list *l1, + struct rcu_pending_list *l2) +{ + if (!l1->head) + l1->head = l2->head; + else + l1->tail->next = l2->head; + l1->tail = l2->tail; + + l2->head = l2->tail = NULL; +} + +static void rcu_pending_list_add(struct rcu_pending_list *l, + struct rcu_head *n) +{ + if (!l->head) + l->head = n; + else + l->tail->next = n; + l->tail = n; + n->next = NULL; +} + +static void merge_expired_lists(struct rcu_pending_pcpu *p) +{ + struct rcu_pending_list *expired = &p->lists[NUM_ACTIVE_RCU_POLL_OLDSTATE]; + + for (struct rcu_pending_list *i = p->lists; i < expired; i++) + if (i->head && __poll_state_synchronize_rcu(p->parent->srcu, i->seq)) + rcu_pending_list_merge(expired, i); +} + +static noinline void __process_finished_items(struct rcu_pending *pending, + struct rcu_pending_pcpu *p, + unsigned long flags) +{ + struct rcu_pending_list *expired = &p->lists[NUM_ACTIVE_RCU_POLL_OLDSTATE]; + struct rcu_pending_seq objs = {}; + struct rcu_head *list = NULL; + + if (p->objs.nr && + __poll_state_synchronize_rcu(pending->srcu, p->objs.data[0].seq)) { + objs = p->objs.data[0]; + darray_remove_item(&p->objs, p->objs.data); + } + + merge_expired_lists(p); + + list = expired->head; + expired->head = expired->tail = NULL; + + spin_unlock_irqrestore(&p->lock, flags); + + switch ((ulong) pending->process) { + case RCU_PENDING_KVFREE: + for (size_t i = 0; i < objs.nr; ) { + size_t nr_this_node = min(GENRADIX_NODE_SIZE / sizeof(void *), objs.nr - i); + + kfree_bulk(nr_this_node, (void **) genradix_ptr(&objs.objs, i)); + i += nr_this_node; + } + genradix_free(&objs.objs); + + while (list) { + struct rcu_head *obj = list; + list = obj->next; + + /* + * low bit of pointer indicates whether rcu_head needs + * to be freed - kvfree_rcu_mightsleep() + */ + BUILD_BUG_ON(ARCH_SLAB_MINALIGN == 0); + + void *ptr = (void *)(((unsigned long) obj->func) & ~1UL); + bool free_head = ((unsigned long) obj->func) & 1UL; + + kvfree(ptr); + if (free_head) + kfree(obj); + } + + break; + + case RCU_PENDING_CALL_RCU: + for (size_t i = 0; i < objs.nr; i++) { + struct rcu_head *obj = *genradix_ptr(&objs.objs, i); + obj->func(obj); + } + genradix_free(&objs.objs); + + while (list) { + struct rcu_head *obj = list; + list = obj->next; + obj->func(obj); + } + break; + + default: + for (size_t i = 0; i < objs.nr; i++) + pending->process(pending, *genradix_ptr(&objs.objs, i)); + genradix_free(&objs.objs); + + while (list) { + struct rcu_head *obj = list; + list = obj->next; + pending->process(pending, obj); + } + break; + } +} + +static bool process_finished_items(struct rcu_pending *pending, + struct rcu_pending_pcpu *p, + unsigned long flags) +{ + /* + * XXX: we should grab the gp seq once and avoid multiple function + * calls, this is called from __rcu_pending_enqueue() fastpath in + * may_sleep==true mode + */ + if ((p->objs.nr && __poll_state_synchronize_rcu(pending->srcu, p->objs.data[0].seq)) || + (p->lists[0].head && __poll_state_synchronize_rcu(pending->srcu, p->lists[0].seq)) || + (p->lists[1].head && __poll_state_synchronize_rcu(pending->srcu, p->lists[1].seq)) || + p->lists[2].head) { + __process_finished_items(pending, p, flags); + return true; + } + + return false; +} + +static void rcu_pending_work(struct work_struct *work) +{ + struct rcu_pending_pcpu *p = + container_of(work, struct rcu_pending_pcpu, work); + struct rcu_pending *pending = p->parent; + unsigned long flags; + + do { + spin_lock_irqsave(&p->lock, flags); + } while (process_finished_items(pending, p, flags)); + + spin_unlock_irqrestore(&p->lock, flags); +} + +static void rcu_pending_rcu_cb(struct rcu_head *rcu) +{ + struct rcu_pending_pcpu *p = container_of(rcu, struct rcu_pending_pcpu, cb); + + schedule_work_on(p->cpu, &p->work); + + unsigned long flags; + spin_lock_irqsave(&p->lock, flags); + if (__rcu_pending_has_pending(p)) + __call_rcu(p->parent->srcu, &p->cb, rcu_pending_rcu_cb); + else + p->cb_armed = false; + spin_unlock_irqrestore(&p->lock, flags); +} + +static __always_inline struct rcu_pending_seq * +get_object_radix(struct rcu_pending_pcpu *p, unsigned long seq) +{ + darray_for_each_reverse(p->objs, objs) + if (objs->seq == seq) + return objs; + + if (darray_push_gfp(&p->objs, ((struct rcu_pending_seq) { .seq = seq }), GFP_ATOMIC)) + return NULL; + + return &darray_last(p->objs); +} + +static noinline bool +rcu_pending_enqueue_list(struct rcu_pending_pcpu *p, unsigned long seq, + struct rcu_head *head, void *ptr, + unsigned long *flags) +{ + if (ptr) { + if (!head) { + /* + * kvfree_rcu_mightsleep(): we weren't passed an + * rcu_head, but we need one: use the low bit of the + * ponter to free to flag that the head needs to be + * freed as well: + */ + ptr = (void *)(((unsigned long) ptr)|1UL); + head = kmalloc(sizeof(*head), __GFP_NOWARN); + if (!head) { + spin_unlock_irqrestore(&p->lock, *flags); + head = kmalloc(sizeof(*head), GFP_KERNEL|__GFP_NOFAIL); + /* + * dropped lock, did GFP_KERNEL allocation, + * check for gp expiration + */ + if (unlikely(__poll_state_synchronize_rcu(p->parent->srcu, seq))) { + kvfree(--ptr); + kfree(head); + spin_lock_irqsave(&p->lock, *flags); + return false; + } + } + } + + head->func = ptr; + } +again: + for (struct rcu_pending_list *i = p->lists; + i < p->lists + NUM_ACTIVE_RCU_POLL_OLDSTATE; i++) { + if (i->seq == seq) { + rcu_pending_list_add(i, head); + return false; + } + } + + for (struct rcu_pending_list *i = p->lists; + i < p->lists + NUM_ACTIVE_RCU_POLL_OLDSTATE; i++) { + if (!i->head) { + i->seq = seq; + rcu_pending_list_add(i, head); + return true; + } + } + + merge_expired_lists(p); + goto again; +} + +/* + * __rcu_pending_enqueue: enqueue a pending RCU item, to be processed (via + * pending->pracess) once grace period elapses. + * + * Attempt to enqueue items onto a radix tree; if memory allocation fails, fall + * back to a linked list. + * + * - If @ptr is NULL, we're enqueuing an item for a generic @pending with a + * process callback + * + * - If @ptr and @head are both not NULL, we're kvfree_rcu() + * + * - If @ptr is not NULL and @head is, we're kvfree_rcu_mightsleep() + * + * - If @may_sleep is true, will do GFP_KERNEL memory allocations and process + * expired items. + */ +static __always_inline void +__rcu_pending_enqueue(struct rcu_pending *pending, struct rcu_head *head, + void *ptr, bool may_sleep) +{ + + struct rcu_pending_pcpu *p; + struct rcu_pending_seq *objs; + struct genradix_node *new_node = NULL; + unsigned long seq, flags; + bool start_gp = false; + + BUG_ON((ptr != NULL) != (pending->process == RCU_PENDING_KVFREE_FN)); + + local_irq_save(flags); + p = this_cpu_ptr(pending->p); + spin_lock(&p->lock); + seq = __get_state_synchronize_rcu(pending->srcu); +restart: + if (may_sleep && + unlikely(process_finished_items(pending, p, flags))) + goto check_expired; + + /* + * In kvfree_rcu() mode, the radix tree is only for slab pointers so + * that we can do kfree_bulk() - vmalloc pointers always use the linked + * list: + */ + if (ptr && unlikely(is_vmalloc_addr(ptr))) + goto list_add; + + objs = get_object_radix(p, seq); + if (unlikely(!objs)) + goto list_add; + + if (unlikely(!objs->cursor)) { + /* + * New radix tree nodes must be added under @p->lock because the + * tree root is in a darray that can be resized (typically, + * genradix supports concurrent unlocked allocation of new + * nodes) - hence preallocation and the retry loop: + */ + objs->cursor = genradix_ptr_alloc_preallocated_inlined(&objs->objs, + objs->nr, &new_node, GFP_ATOMIC|__GFP_NOWARN); + if (unlikely(!objs->cursor)) { + if (may_sleep) { + spin_unlock_irqrestore(&p->lock, flags); + + gfp_t gfp = GFP_KERNEL; + if (!head) + gfp |= __GFP_NOFAIL; + + new_node = genradix_alloc_node(gfp); + if (!new_node) + may_sleep = false; + goto check_expired; + } +list_add: + start_gp = rcu_pending_enqueue_list(p, seq, head, ptr, &flags); + goto start_gp; + } + } + + *objs->cursor++ = ptr ?: head; + /* zero cursor if we hit the end of a radix tree node: */ + if (!(((ulong) objs->cursor) & (GENRADIX_NODE_SIZE - 1))) + objs->cursor = NULL; + start_gp = !objs->nr; + objs->nr++; +start_gp: + if (unlikely(start_gp)) { + /* + * We only have one callback (ideally, we would have one for + * every outstanding graceperiod) - so if our callback is + * already in flight, we may still have to start a grace period + * (since we used get_state() above, not start_poll()) + */ + if (!p->cb_armed) { + p->cb_armed = true; + __call_rcu(pending->srcu, &p->cb, rcu_pending_rcu_cb); + } else { + __start_poll_synchronize_rcu(pending->srcu); + } + } + spin_unlock_irqrestore(&p->lock, flags); +free_node: + if (new_node) + genradix_free_node(new_node); + return; +check_expired: + if (unlikely(__poll_state_synchronize_rcu(pending->srcu, seq))) { + switch ((ulong) pending->process) { + case RCU_PENDING_KVFREE: + kvfree(ptr); + break; + case RCU_PENDING_CALL_RCU: + head->func(head); + break; + default: + pending->process(pending, head); + break; + } + goto free_node; + } + + local_irq_save(flags); + p = this_cpu_ptr(pending->p); + spin_lock(&p->lock); + goto restart; +} + +void rcu_pending_enqueue(struct rcu_pending *pending, struct rcu_head *obj) +{ + __rcu_pending_enqueue(pending, obj, NULL, true); +} + +static struct rcu_head *rcu_pending_pcpu_dequeue(struct rcu_pending_pcpu *p) +{ + struct rcu_head *ret = NULL; + + spin_lock_irq(&p->lock); + darray_for_each(p->objs, objs) + if (objs->nr) { + ret = *genradix_ptr(&objs->objs, --objs->nr); + objs->cursor = NULL; + if (!objs->nr) + genradix_free(&objs->objs); + goto out; + } + + static_array_for_each(p->lists, i) + if (i->head) { + ret = i->head; + i->head = ret->next; + if (!i->head) + i->tail = NULL; + goto out; + } +out: + spin_unlock_irq(&p->lock); + + return ret; +} + +struct rcu_head *rcu_pending_dequeue(struct rcu_pending *pending) +{ + return rcu_pending_pcpu_dequeue(raw_cpu_ptr(pending->p)); +} + +struct rcu_head *rcu_pending_dequeue_from_all(struct rcu_pending *pending) +{ + struct rcu_head *ret = rcu_pending_dequeue(pending); + + if (ret) + return ret; + + int cpu; + for_each_possible_cpu(cpu) { + ret = rcu_pending_pcpu_dequeue(per_cpu_ptr(pending->p, cpu)); + if (ret) + break; + } + return ret; +} + +static bool rcu_pending_has_pending_or_armed(struct rcu_pending *pending) +{ + int cpu; + for_each_possible_cpu(cpu) { + struct rcu_pending_pcpu *p = per_cpu_ptr(pending->p, cpu); + spin_lock_irq(&p->lock); + if (__rcu_pending_has_pending(p) || p->cb_armed) { + spin_unlock_irq(&p->lock); + return true; + } + spin_unlock_irq(&p->lock); + } + + return false; +} + +void rcu_pending_exit(struct rcu_pending *pending) +{ + int cpu; + + if (!pending->p) + return; + + while (rcu_pending_has_pending_or_armed(pending)) { + __rcu_barrier(pending->srcu); + + for_each_possible_cpu(cpu) { + struct rcu_pending_pcpu *p = per_cpu_ptr(pending->p, cpu); + flush_work(&p->work); + } + } + + for_each_possible_cpu(cpu) { + struct rcu_pending_pcpu *p = per_cpu_ptr(pending->p, cpu); + flush_work(&p->work); + } + + for_each_possible_cpu(cpu) { + struct rcu_pending_pcpu *p = per_cpu_ptr(pending->p, cpu); + + static_array_for_each(p->lists, i) + WARN_ON(i->head); + WARN_ON(p->objs.nr); + darray_exit(&p->objs); + } + free_percpu(pending->p); +} + +/** + * rcu_pending_init: - initialize a rcu_pending + * + * @pending: Object to init + * @srcu: May optionally be used with an srcu_struct; if NULL, uses normal + * RCU flavor + * @process: Callback function invoked on objects once their RCU barriers + * have completed; if NULL, kvfree() is used. + */ +int rcu_pending_init(struct rcu_pending *pending, + struct srcu_struct *srcu, + rcu_pending_process_fn process) +{ + pending->p = alloc_percpu(struct rcu_pending_pcpu); + if (!pending->p) + return -ENOMEM; + + int cpu; + for_each_possible_cpu(cpu) { + struct rcu_pending_pcpu *p = per_cpu_ptr(pending->p, cpu); + p->parent = pending; + p->cpu = cpu; + spin_lock_init(&p->lock); + darray_init(&p->objs); + INIT_WORK(&p->work, rcu_pending_work); + } + + pending->srcu = srcu; + pending->process = process; + + return 0; +} diff --git a/fs/bcachefs/rcu_pending.h b/fs/bcachefs/rcu_pending.h new file mode 100644 index 000000000000..a875c640da8d --- /dev/null +++ b/fs/bcachefs/rcu_pending.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_RCU_PENDING_H +#define _LINUX_RCU_PENDING_H + +struct rcu_pending; +typedef void (*rcu_pending_process_fn)(struct rcu_pending *, struct rcu_head *); + +struct rcu_pending_pcpu; + +struct rcu_pending { + struct rcu_pending_pcpu __percpu *p; + struct srcu_struct *srcu; + rcu_pending_process_fn process; +}; + +void rcu_pending_enqueue(struct rcu_pending *pending, struct rcu_head *obj); +struct rcu_head *rcu_pending_dequeue(struct rcu_pending *pending); +struct rcu_head *rcu_pending_dequeue_from_all(struct rcu_pending *pending); + +void rcu_pending_exit(struct rcu_pending *pending); +int rcu_pending_init(struct rcu_pending *pending, + struct srcu_struct *srcu, + rcu_pending_process_fn process); + +#endif /* _LINUX_RCU_PENDING_H */ From d2ed0f206a9d8b3add18879f39dd128ff55e4d77 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Fri, 23 Aug 2024 18:21:31 -0400 Subject: [PATCH 129/302] bcachefs: rcu_pending now works in userspace Signed-off-by: Kent Overstreet --- fs/bcachefs/rcu_pending.c | 55 ++++++++++++++++++++++++++++++++++++--- fs/bcachefs/rcu_pending.h | 2 ++ 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/fs/bcachefs/rcu_pending.c b/fs/bcachefs/rcu_pending.c index 8f8d914d3998..40a20192eee8 100644 --- a/fs/bcachefs/rcu_pending.c +++ b/fs/bcachefs/rcu_pending.c @@ -115,24 +115,40 @@ static bool __rcu_pending_has_pending(struct rcu_pending_pcpu *p) static void rcu_pending_list_merge(struct rcu_pending_list *l1, struct rcu_pending_list *l2) { +#ifdef __KERNEL__ if (!l1->head) l1->head = l2->head; else l1->tail->next = l2->head; - l1->tail = l2->tail; +#else + if (!l1->head) + l1->head = l2->head; + else + l1->tail->next.next = (void *) l2->head; +#endif + l1->tail = l2->tail; l2->head = l2->tail = NULL; } static void rcu_pending_list_add(struct rcu_pending_list *l, struct rcu_head *n) { +#ifdef __KERNEL__ if (!l->head) l->head = n; else l->tail->next = n; l->tail = n; n->next = NULL; +#else + if (!l->head) + l->head = n; + else + l->tail->next.next = (void *) n; + l->tail = n; + n->next.next = NULL; +#endif } static void merge_expired_lists(struct rcu_pending_pcpu *p) @@ -144,6 +160,19 @@ static void merge_expired_lists(struct rcu_pending_pcpu *p) rcu_pending_list_merge(expired, i); } +#ifndef __KERNEL__ +static inline void kfree_bulk(size_t nr, void ** p) +{ + while (nr--) + kfree(*p); +} + +#define local_irq_save(flags) \ +do { \ + flags = 0; \ +} while (0) +#endif + static noinline void __process_finished_items(struct rcu_pending *pending, struct rcu_pending_pcpu *p, unsigned long flags) @@ -177,7 +206,11 @@ static noinline void __process_finished_items(struct rcu_pending *pending, while (list) { struct rcu_head *obj = list; +#ifdef __KERNEL__ list = obj->next; +#else + list = (void *) obj->next.next; +#endif /* * low bit of pointer indicates whether rcu_head needs @@ -204,7 +237,11 @@ static noinline void __process_finished_items(struct rcu_pending *pending, while (list) { struct rcu_head *obj = list; +#ifdef __KERNEL__ list = obj->next; +#else + list = (void *) obj->next.next; +#endif obj->func(obj); } break; @@ -216,7 +253,11 @@ static noinline void __process_finished_items(struct rcu_pending *pending, while (list) { struct rcu_head *obj = list; +#ifdef __KERNEL__ list = obj->next; +#else + list = (void *) obj->next.next; +#endif pending->process(pending, obj); } break; @@ -265,11 +306,13 @@ static void rcu_pending_rcu_cb(struct rcu_head *rcu) unsigned long flags; spin_lock_irqsave(&p->lock, flags); - if (__rcu_pending_has_pending(p)) + if (__rcu_pending_has_pending(p)) { + spin_unlock_irqrestore(&p->lock, flags); __call_rcu(p->parent->srcu, &p->cb, rcu_pending_rcu_cb); - else + } else { p->cb_armed = false; - spin_unlock_irqrestore(&p->lock, flags); + spin_unlock_irqrestore(&p->lock, flags); + } } static __always_inline struct rcu_pending_seq * @@ -489,7 +532,11 @@ static struct rcu_head *rcu_pending_pcpu_dequeue(struct rcu_pending_pcpu *p) static_array_for_each(p->lists, i) if (i->head) { ret = i->head; +#ifdef __KERNEL__ i->head = ret->next; +#else + i->head = (void *) ret->next.next; +#endif if (!i->head) i->tail = NULL; goto out; diff --git a/fs/bcachefs/rcu_pending.h b/fs/bcachefs/rcu_pending.h index a875c640da8d..71a2f4ddaade 100644 --- a/fs/bcachefs/rcu_pending.h +++ b/fs/bcachefs/rcu_pending.h @@ -2,6 +2,8 @@ #ifndef _LINUX_RCU_PENDING_H #define _LINUX_RCU_PENDING_H +#include + struct rcu_pending; typedef void (*rcu_pending_process_fn)(struct rcu_pending *, struct rcu_head *); From f2bfe7e83765f3bd84382cc75d8ac3ca619de39a Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sat, 8 Jun 2024 22:32:40 -0400 Subject: [PATCH 130/302] bcachefs: Rip out freelists from btree key cache Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_key_cache.c | 369 +++++----------------------- fs/bcachefs/btree_key_cache_types.h | 14 -- fs/bcachefs/btree_types.h | 4 +- 3 files changed, 57 insertions(+), 330 deletions(-) diff --git a/fs/bcachefs/btree_key_cache.c b/fs/bcachefs/btree_key_cache.c index fda7998734cb..dfaeb0810c5e 100644 --- a/fs/bcachefs/btree_key_cache.c +++ b/fs/bcachefs/btree_key_cache.c @@ -79,14 +79,24 @@ static bool bkey_cached_lock_for_evict(struct bkey_cached *ck) return true; } -static void bkey_cached_evict(struct btree_key_cache *c, +static bool bkey_cached_evict(struct btree_key_cache *c, struct bkey_cached *ck) { - BUG_ON(rhashtable_remove_fast(&c->table, &ck->hash, - bch2_btree_key_cache_params)); - memset(&ck->key, ~0, sizeof(ck->key)); + bool ret = !rhashtable_remove_fast(&c->table, &ck->hash, + bch2_btree_key_cache_params); + if (ret) { + memset(&ck->key, ~0, sizeof(ck->key)); + atomic_long_dec(&c->nr_keys); + } - atomic_long_dec(&c->nr_keys); + return ret; +} + +static void __bkey_cached_free(struct rcu_head *rcu) +{ + struct bkey_cached *ck = container_of(rcu, struct bkey_cached, rcu); + + kmem_cache_free(bch2_key_cache, ck); } static void bkey_cached_free(struct btree_key_cache *bc, @@ -94,115 +104,14 @@ static void bkey_cached_free(struct btree_key_cache *bc, { struct bch_fs *c = container_of(bc, struct bch_fs, btree_key_cache); - BUG_ON(test_bit(BKEY_CACHED_DIRTY, &ck->flags)); - - ck->btree_trans_barrier_seq = - start_poll_synchronize_srcu(&c->btree_trans_barrier); - - if (ck->c.lock.readers) { - list_move_tail(&ck->list, &bc->freed_pcpu); - bc->nr_freed_pcpu++; - } else { - list_move_tail(&ck->list, &bc->freed_nonpcpu); - bc->nr_freed_nonpcpu++; - } - atomic_long_inc(&bc->nr_freed); - kfree(ck->k); ck->k = NULL; ck->u64s = 0; six_unlock_write(&ck->c.lock); six_unlock_intent(&ck->c.lock); -} -#ifdef __KERNEL__ -static void __bkey_cached_move_to_freelist_ordered(struct btree_key_cache *bc, - struct bkey_cached *ck) -{ - struct bkey_cached *pos; - - bc->nr_freed_nonpcpu++; - - list_for_each_entry_reverse(pos, &bc->freed_nonpcpu, list) { - if (ULONG_CMP_GE(ck->btree_trans_barrier_seq, - pos->btree_trans_barrier_seq)) { - list_move(&ck->list, &pos->list); - return; - } - } - - list_move(&ck->list, &bc->freed_nonpcpu); -} -#endif - -static void bkey_cached_move_to_freelist(struct btree_key_cache *bc, - struct bkey_cached *ck) -{ - BUG_ON(test_bit(BKEY_CACHED_DIRTY, &ck->flags)); - - if (!ck->c.lock.readers) { -#ifdef __KERNEL__ - struct btree_key_cache_freelist *f; - bool freed = false; - - preempt_disable(); - f = this_cpu_ptr(bc->pcpu_freed); - - if (f->nr < ARRAY_SIZE(f->objs)) { - f->objs[f->nr++] = ck; - freed = true; - } - preempt_enable(); - - if (!freed) { - mutex_lock(&bc->lock); - preempt_disable(); - f = this_cpu_ptr(bc->pcpu_freed); - - while (f->nr > ARRAY_SIZE(f->objs) / 2) { - struct bkey_cached *ck2 = f->objs[--f->nr]; - - __bkey_cached_move_to_freelist_ordered(bc, ck2); - } - preempt_enable(); - - __bkey_cached_move_to_freelist_ordered(bc, ck); - mutex_unlock(&bc->lock); - } -#else - mutex_lock(&bc->lock); - list_move_tail(&ck->list, &bc->freed_nonpcpu); - bc->nr_freed_nonpcpu++; - mutex_unlock(&bc->lock); -#endif - } else { - mutex_lock(&bc->lock); - list_move_tail(&ck->list, &bc->freed_pcpu); - bc->nr_freed_pcpu++; - mutex_unlock(&bc->lock); - } -} - -static void bkey_cached_free_fast(struct btree_key_cache *bc, - struct bkey_cached *ck) -{ - struct bch_fs *c = container_of(bc, struct bch_fs, btree_key_cache); - - ck->btree_trans_barrier_seq = - start_poll_synchronize_srcu(&c->btree_trans_barrier); - - list_del_init(&ck->list); - atomic_long_inc(&bc->nr_freed); - - kfree(ck->k); - ck->k = NULL; - ck->u64s = 0; - - bkey_cached_move_to_freelist(bc, ck); - - six_unlock_write(&ck->c.lock); - six_unlock_intent(&ck->c.lock); + call_srcu(&c->btree_trans_barrier, &ck->rcu, __bkey_cached_free); } static struct bkey_cached *__bkey_cached_alloc(unsigned key_u64s, gfp_t gfp) @@ -222,78 +131,10 @@ static struct bkey_cached *__bkey_cached_alloc(unsigned key_u64s, gfp_t gfp) static struct bkey_cached * bkey_cached_alloc(struct btree_trans *trans, struct btree_path *path, unsigned key_u64s) { - struct bch_fs *c = trans->c; - struct btree_key_cache *bc = &c->btree_key_cache; - struct bkey_cached *ck = NULL; bool pcpu_readers = btree_uses_pcpu_readers(path->btree_id); int ret; - if (!pcpu_readers) { -#ifdef __KERNEL__ - struct btree_key_cache_freelist *f; - - preempt_disable(); - f = this_cpu_ptr(bc->pcpu_freed); - if (f->nr) - ck = f->objs[--f->nr]; - preempt_enable(); - - if (!ck) { - mutex_lock(&bc->lock); - preempt_disable(); - f = this_cpu_ptr(bc->pcpu_freed); - - while (!list_empty(&bc->freed_nonpcpu) && - f->nr < ARRAY_SIZE(f->objs) / 2) { - ck = list_last_entry(&bc->freed_nonpcpu, struct bkey_cached, list); - list_del_init(&ck->list); - bc->nr_freed_nonpcpu--; - f->objs[f->nr++] = ck; - } - - ck = f->nr ? f->objs[--f->nr] : NULL; - preempt_enable(); - mutex_unlock(&bc->lock); - } -#else - mutex_lock(&bc->lock); - if (!list_empty(&bc->freed_nonpcpu)) { - ck = list_last_entry(&bc->freed_nonpcpu, struct bkey_cached, list); - list_del_init(&ck->list); - bc->nr_freed_nonpcpu--; - } - mutex_unlock(&bc->lock); -#endif - } else { - mutex_lock(&bc->lock); - if (!list_empty(&bc->freed_pcpu)) { - ck = list_last_entry(&bc->freed_pcpu, struct bkey_cached, list); - list_del_init(&ck->list); - bc->nr_freed_pcpu--; - } - mutex_unlock(&bc->lock); - } - - if (ck) { - ret = btree_node_lock_nopath(trans, &ck->c, SIX_LOCK_intent, _THIS_IP_); - if (unlikely(ret)) { - bkey_cached_move_to_freelist(bc, ck); - return ERR_PTR(ret); - } - - btree_path_cached_set(trans, path, ck, BTREE_NODE_INTENT_LOCKED); - - ret = bch2_btree_node_lock_write(trans, path, &ck->c); - if (unlikely(ret)) { - btree_node_unlock(trans, path, 0); - bkey_cached_move_to_freelist(bc, ck); - return ERR_PTR(ret); - } - - return ck; - } - - ck = allocate_dropping_locks(trans, ret, + struct bkey_cached *ck = allocate_dropping_locks(trans, ret, __bkey_cached_alloc(key_u64s, _gfp)); if (ret) { if (ck) @@ -305,7 +146,6 @@ bkey_cached_alloc(struct btree_trans *trans, struct btree_path *path, unsigned k if (!ck) return NULL; - INIT_LIST_HEAD(&ck->list); bch2_btree_lock_init(&ck->c, pcpu_readers ? SIX_LOCK_INIT_PCPU : 0); ck->c.cached = true; @@ -322,21 +162,21 @@ bkey_cached_reuse(struct btree_key_cache *c) struct bkey_cached *ck; unsigned i; - mutex_lock(&c->lock); rcu_read_lock(); tbl = rht_dereference_rcu(c->table.tbl, &c->table); for (i = 0; i < tbl->size; i++) rht_for_each_entry_rcu(ck, pos, tbl, i, hash) { if (!test_bit(BKEY_CACHED_DIRTY, &ck->flags) && bkey_cached_lock_for_evict(ck)) { - bkey_cached_evict(c, ck); - goto out; + if (bkey_cached_evict(c, ck)) + goto out; + six_unlock_write(&ck->c.lock); + six_unlock_intent(&ck->c.lock); } } ck = NULL; out: rcu_read_unlock(); - mutex_unlock(&c->lock); return ck; } @@ -415,7 +255,7 @@ static int btree_key_cache_create(struct btree_trans *trans, struct btree_path * path->uptodate = BTREE_ITER_UPTODATE; return 0; err: - bkey_cached_free_fast(bc, ck); + bkey_cached_free(bc, ck); mark_btree_node_locked_noreset(path, 0, BTREE_NODE_UNLOCKED); return ret; @@ -611,8 +451,12 @@ evict: } mark_btree_node_locked_noreset(path, 0, BTREE_NODE_UNLOCKED); - bkey_cached_evict(&c->btree_key_cache, ck); - bkey_cached_free_fast(&c->btree_key_cache, ck); + if (bkey_cached_evict(&c->btree_key_cache, ck)) { + bkey_cached_free(&c->btree_key_cache, ck); + } else { + six_unlock_write(&ck->c.lock); + six_unlock_intent(&ck->c.lock); + } } out: bch2_trans_iter_exit(trans, &b_iter); @@ -722,7 +566,7 @@ void bch2_btree_key_cache_drop(struct btree_trans *trans, } bkey_cached_evict(bc, ck); - bkey_cached_free_fast(bc, ck); + bkey_cached_free(bc, ck); mark_btree_node_locked(trans, path, 0, BTREE_NODE_UNLOCKED); btree_path_set_dirty(path, BTREE_ITER_NEED_TRAVERSE); @@ -735,48 +579,14 @@ static unsigned long bch2_btree_key_cache_scan(struct shrinker *shrink, struct bch_fs *c = shrink->private_data; struct btree_key_cache *bc = &c->btree_key_cache; struct bucket_table *tbl; - struct bkey_cached *ck, *t; + struct bkey_cached *ck; size_t scanned = 0, freed = 0, nr = sc->nr_to_scan; - unsigned start, flags; + unsigned iter, start; int srcu_idx; - mutex_lock(&bc->lock); - bc->requested_to_free += sc->nr_to_scan; - srcu_idx = srcu_read_lock(&c->btree_trans_barrier); - flags = memalloc_nofs_save(); - - /* - * Newest freed entries are at the end of the list - once we hit one - * that's too new to be freed, we can bail out: - */ - list_for_each_entry_safe(ck, t, &bc->freed_nonpcpu, list) { - if (!poll_state_synchronize_srcu(&c->btree_trans_barrier, - ck->btree_trans_barrier_seq)) - break; - - list_del(&ck->list); - six_lock_exit(&ck->c.lock); - kmem_cache_free(bch2_key_cache, ck); - atomic_long_dec(&bc->nr_freed); - bc->nr_freed_nonpcpu--; - bc->freed++; - } - - list_for_each_entry_safe(ck, t, &bc->freed_pcpu, list) { - if (!poll_state_synchronize_srcu(&c->btree_trans_barrier, - ck->btree_trans_barrier_seq)) - break; - - list_del(&ck->list); - six_lock_exit(&ck->c.lock); - kmem_cache_free(bch2_key_cache, ck); - atomic_long_dec(&bc->nr_freed); - bc->nr_freed_pcpu--; - bc->freed++; - } - rcu_read_lock(); + tbl = rht_dereference_rcu(bc->table.tbl, &bc->table); /* @@ -792,17 +602,18 @@ static unsigned long bch2_btree_key_cache_scan(struct shrinker *shrink, return SHRINK_STOP; } - if (bc->shrink_iter >= tbl->size) - bc->shrink_iter = 0; - start = bc->shrink_iter; + iter = bc->shrink_iter; + if (iter >= tbl->size) + iter = 0; + start = iter; do { struct rhash_head *pos, *next; - pos = rht_ptr_rcu(&tbl->buckets[bc->shrink_iter]); + pos = rht_ptr_rcu(&tbl->buckets[iter]); while (!rht_is_a_nulls(pos)) { - next = rht_dereference_bucket_rcu(pos->next, tbl, bc->shrink_iter); + next = rht_dereference_bucket_rcu(pos->next, tbl, iter); ck = container_of(pos, struct bkey_cached, hash); if (test_bit(BKEY_CACHED_DIRTY, &ck->flags)) { @@ -812,29 +623,31 @@ static unsigned long bch2_btree_key_cache_scan(struct shrinker *shrink, bc->skipped_accessed++; } else if (!bkey_cached_lock_for_evict(ck)) { bc->skipped_lock_fail++; - } else { - bkey_cached_evict(bc, ck); + } else if (bkey_cached_evict(bc, ck)) { bkey_cached_free(bc, ck); - bc->moved_to_freelist++; + bc->freed++; freed++; + } else { + six_unlock_write(&ck->c.lock); + six_unlock_intent(&ck->c.lock); } scanned++; if (scanned >= nr) - break; + goto out; pos = next; } - bc->shrink_iter++; - if (bc->shrink_iter >= tbl->size) - bc->shrink_iter = 0; - } while (scanned < nr && bc->shrink_iter != start); + iter++; + if (iter >= tbl->size) + iter = 0; + } while (scanned < nr && iter != start); +out: + bc->shrink_iter = iter; rcu_read_unlock(); - memalloc_nofs_restore(flags); srcu_read_unlock(&c->btree_trans_barrier, srcu_idx); - mutex_unlock(&bc->lock); return freed; } @@ -862,18 +675,13 @@ void bch2_fs_btree_key_cache_exit(struct btree_key_cache *bc) { struct bch_fs *c = container_of(bc, struct bch_fs, btree_key_cache); struct bucket_table *tbl; - struct bkey_cached *ck, *n; + struct bkey_cached *ck; struct rhash_head *pos; LIST_HEAD(items); unsigned i; -#ifdef __KERNEL__ - int cpu; -#endif shrinker_free(bc->shrink); - mutex_lock(&bc->lock); - /* * The loop is needed to guard against racing with rehash: */ @@ -892,44 +700,14 @@ void bch2_fs_btree_key_cache_exit(struct btree_key_cache *bc) for (i = 0; i < tbl->size; i++) while (pos = rht_ptr_rcu(&tbl->buckets[i]), !rht_is_a_nulls(pos)) { ck = container_of(pos, struct bkey_cached, hash); - bkey_cached_evict(bc, ck); - list_add(&ck->list, &items); + BUG_ON(!bkey_cached_evict(bc, ck)); + kfree(ck->k); + kmem_cache_free(bch2_key_cache, ck); } } rcu_read_unlock(); } -#ifdef __KERNEL__ - if (bc->pcpu_freed) { - for_each_possible_cpu(cpu) { - struct btree_key_cache_freelist *f = - per_cpu_ptr(bc->pcpu_freed, cpu); - - for (i = 0; i < f->nr; i++) { - ck = f->objs[i]; - list_add(&ck->list, &items); - } - } - } -#endif - - BUG_ON(list_count_nodes(&bc->freed_pcpu) != bc->nr_freed_pcpu); - BUG_ON(list_count_nodes(&bc->freed_nonpcpu) != bc->nr_freed_nonpcpu); - - list_splice(&bc->freed_pcpu, &items); - list_splice(&bc->freed_nonpcpu, &items); - - mutex_unlock(&bc->lock); - - list_for_each_entry_safe(ck, n, &items, list) { - cond_resched(); - - list_del(&ck->list); - kfree(ck->k); - six_lock_exit(&ck->c.lock); - kmem_cache_free(bch2_key_cache, ck); - } - if (atomic_long_read(&bc->nr_dirty) && !bch2_journal_error(&c->journal) && test_bit(BCH_FS_was_rw, &c->flags)) @@ -942,15 +720,10 @@ void bch2_fs_btree_key_cache_exit(struct btree_key_cache *bc) if (bc->table_init_done) rhashtable_destroy(&bc->table); - - free_percpu(bc->pcpu_freed); } void bch2_fs_btree_key_cache_init_early(struct btree_key_cache *c) { - mutex_init(&c->lock); - INIT_LIST_HEAD(&c->freed_pcpu); - INIT_LIST_HEAD(&c->freed_nonpcpu); } int bch2_fs_btree_key_cache_init(struct btree_key_cache *bc) @@ -958,12 +731,6 @@ int bch2_fs_btree_key_cache_init(struct btree_key_cache *bc) struct bch_fs *c = container_of(bc, struct bch_fs, btree_key_cache); struct shrinker *shrink; -#ifdef __KERNEL__ - bc->pcpu_freed = alloc_percpu(struct btree_key_cache_freelist); - if (!bc->pcpu_freed) - return -BCH_ERR_ENOMEM_fs_btree_cache_init; -#endif - if (rhashtable_init(&bc->table, &bch2_btree_key_cache_params)) return -BCH_ERR_ENOMEM_fs_btree_cache_init; @@ -984,45 +751,19 @@ int bch2_fs_btree_key_cache_init(struct btree_key_cache *bc) void bch2_btree_key_cache_to_text(struct printbuf *out, struct btree_key_cache *bc) { - struct bch_fs *c = container_of(bc, struct bch_fs, btree_key_cache); - printbuf_tabstop_push(out, 24); printbuf_tabstop_push(out, 12); - unsigned flags = memalloc_nofs_save(); - mutex_lock(&bc->lock); prt_printf(out, "keys:\t%lu\r\n", atomic_long_read(&bc->nr_keys)); prt_printf(out, "dirty:\t%lu\r\n", atomic_long_read(&bc->nr_dirty)); - prt_printf(out, "freelist:\t%lu\r\n", atomic_long_read(&bc->nr_freed)); - prt_printf(out, "nonpcpu freelist:\t%zu\r\n", bc->nr_freed_nonpcpu); - prt_printf(out, "pcpu freelist:\t%zu\r\n", bc->nr_freed_pcpu); + prt_printf(out, "table size:\t%u\r\n", bc->table.tbl->size); prt_printf(out, "\nshrinker:\n"); prt_printf(out, "requested_to_free:\t%lu\r\n", bc->requested_to_free); prt_printf(out, "freed:\t%lu\r\n", bc->freed); - prt_printf(out, "moved_to_freelist:\t%lu\r\n", bc->moved_to_freelist); prt_printf(out, "skipped_dirty:\t%lu\r\n", bc->skipped_dirty); prt_printf(out, "skipped_accessed:\t%lu\r\n", bc->skipped_accessed); prt_printf(out, "skipped_lock_fail:\t%lu\r\n", bc->skipped_lock_fail); - - prt_printf(out, "srcu seq:\t%lu\r\n", get_state_synchronize_srcu(&c->btree_trans_barrier)); - - struct bkey_cached *ck; - unsigned iter = 0; - list_for_each_entry(ck, &bc->freed_nonpcpu, list) { - prt_printf(out, "freed_nonpcpu:\t%lu\r\n", ck->btree_trans_barrier_seq); - if (++iter > 10) - break; - } - - iter = 0; - list_for_each_entry(ck, &bc->freed_pcpu, list) { - prt_printf(out, "freed_pcpu:\t%lu\r\n", ck->btree_trans_barrier_seq); - if (++iter > 10) - break; - } - mutex_unlock(&bc->lock); - memalloc_flags_restore(flags); } void bch2_btree_key_cache_exit(void) diff --git a/fs/bcachefs/btree_key_cache_types.h b/fs/bcachefs/btree_key_cache_types.h index 237e8bb3ac40..e026c65f54e1 100644 --- a/fs/bcachefs/btree_key_cache_types.h +++ b/fs/bcachefs/btree_key_cache_types.h @@ -2,33 +2,19 @@ #ifndef _BCACHEFS_BTREE_KEY_CACHE_TYPES_H #define _BCACHEFS_BTREE_KEY_CACHE_TYPES_H -struct btree_key_cache_freelist { - struct bkey_cached *objs[16]; - unsigned nr; -}; - struct btree_key_cache { - struct mutex lock; struct rhashtable table; bool table_init_done; - struct list_head freed_pcpu; - size_t nr_freed_pcpu; - struct list_head freed_nonpcpu; - size_t nr_freed_nonpcpu; - struct shrinker *shrink; unsigned shrink_iter; - struct btree_key_cache_freelist __percpu *pcpu_freed; - atomic_long_t nr_freed; atomic_long_t nr_keys; atomic_long_t nr_dirty; /* shrinker stats */ unsigned long requested_to_free; unsigned long freed; - unsigned long moved_to_freelist; unsigned long skipped_dirty; unsigned long skipped_accessed; unsigned long skipped_lock_fail; diff --git a/fs/bcachefs/btree_types.h b/fs/bcachefs/btree_types.h index b256b2a20a4f..bca56b6359e7 100644 --- a/fs/bcachefs/btree_types.h +++ b/fs/bcachefs/btree_types.h @@ -386,17 +386,17 @@ struct bkey_cached { struct btree_bkey_cached_common c; unsigned long flags; - unsigned long btree_trans_barrier_seq; u16 u64s; struct bkey_cached_key key; struct rhash_head hash; - struct list_head list; struct journal_entry_pin journal; u64 seq; struct bkey_i *k; + + struct rcu_head rcu; }; static inline struct bpos btree_node_pos(struct btree_bkey_cached_common *b) From 5f1929f1f077d3997ab8cd4a8136aad304b0e9df Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 13 Jun 2024 15:35:47 -0400 Subject: [PATCH 131/302] bcachefs: key cache can now allocate from pending btree_trans objects can hold the btree_trans_barrier srcu read lock for an extended amount of time (they shouldn't, but it's difficult to guarantee). the srcu barrier blocks memory reclaim, so to avoid too many stranded key cache items, this uses the new pending_rcu_items to allocate from pending items - like we did before, but now without a global lock on the key cache. Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_key_cache.c | 58 ++++++++++++++++++++++------- fs/bcachefs/btree_key_cache_types.h | 6 +++ fs/bcachefs/btree_types.h | 1 - 3 files changed, 50 insertions(+), 15 deletions(-) diff --git a/fs/bcachefs/btree_key_cache.c b/fs/bcachefs/btree_key_cache.c index dfaeb0810c5e..3048adde9284 100644 --- a/fs/bcachefs/btree_key_cache.c +++ b/fs/bcachefs/btree_key_cache.c @@ -92,18 +92,18 @@ static bool bkey_cached_evict(struct btree_key_cache *c, return ret; } -static void __bkey_cached_free(struct rcu_head *rcu) +static void __bkey_cached_free(struct rcu_pending *pending, struct rcu_head *rcu) { + struct bch_fs *c = container_of(pending->srcu, struct bch_fs, btree_trans_barrier); struct bkey_cached *ck = container_of(rcu, struct bkey_cached, rcu); + this_cpu_dec(*c->btree_key_cache.nr_pending); kmem_cache_free(bch2_key_cache, ck); } static void bkey_cached_free(struct btree_key_cache *bc, struct bkey_cached *ck) { - struct bch_fs *c = container_of(bc, struct bch_fs, btree_key_cache); - kfree(ck->k); ck->k = NULL; ck->u64s = 0; @@ -111,7 +111,9 @@ static void bkey_cached_free(struct btree_key_cache *bc, six_unlock_write(&ck->c.lock); six_unlock_intent(&ck->c.lock); - call_srcu(&c->btree_trans_barrier, &ck->rcu, __bkey_cached_free); + bool pcpu_readers = ck->c.lock.readers != NULL; + rcu_pending_enqueue(&bc->pending[pcpu_readers], &ck->rcu); + this_cpu_inc(*bc->nr_pending); } static struct bkey_cached *__bkey_cached_alloc(unsigned key_u64s, gfp_t gfp) @@ -131,10 +133,18 @@ static struct bkey_cached *__bkey_cached_alloc(unsigned key_u64s, gfp_t gfp) static struct bkey_cached * bkey_cached_alloc(struct btree_trans *trans, struct btree_path *path, unsigned key_u64s) { + struct bch_fs *c = trans->c; + struct btree_key_cache *bc = &c->btree_key_cache; bool pcpu_readers = btree_uses_pcpu_readers(path->btree_id); int ret; - struct bkey_cached *ck = allocate_dropping_locks(trans, ret, + struct bkey_cached *ck = container_of_or_null( + rcu_pending_dequeue(&bc->pending[pcpu_readers]), + struct bkey_cached, rcu); + if (ck) + goto lock; + + ck = allocate_dropping_locks(trans, ret, __bkey_cached_alloc(key_u64s, _gfp)); if (ret) { if (ck) @@ -143,14 +153,19 @@ bkey_cached_alloc(struct btree_trans *trans, struct btree_path *path, unsigned k return ERR_PTR(ret); } - if (!ck) - return NULL; + if (ck) { + bch2_btree_lock_init(&ck->c, pcpu_readers ? SIX_LOCK_INIT_PCPU : 0); + ck->c.cached = true; + goto lock; + } - bch2_btree_lock_init(&ck->c, pcpu_readers ? SIX_LOCK_INIT_PCPU : 0); - - ck->c.cached = true; - BUG_ON(!six_trylock_intent(&ck->c.lock)); - BUG_ON(!six_trylock_write(&ck->c.lock)); + ck = container_of_or_null(rcu_pending_dequeue_from_all(&bc->pending[pcpu_readers]), + struct bkey_cached, rcu); + if (ck) + goto lock; +lock: + six_lock_intent(&ck->c.lock, NULL, NULL); + six_lock_write(&ck->c.lock, NULL, NULL); return ck; } @@ -720,6 +735,11 @@ void bch2_fs_btree_key_cache_exit(struct btree_key_cache *bc) if (bc->table_init_done) rhashtable_destroy(&bc->table); + + rcu_pending_exit(&bc->pending[0]); + rcu_pending_exit(&bc->pending[1]); + + free_percpu(bc->nr_pending); } void bch2_fs_btree_key_cache_init_early(struct btree_key_cache *c) @@ -731,6 +751,14 @@ int bch2_fs_btree_key_cache_init(struct btree_key_cache *bc) struct bch_fs *c = container_of(bc, struct bch_fs, btree_key_cache); struct shrinker *shrink; + bc->nr_pending = alloc_percpu(size_t); + if (!bc->nr_pending) + return -BCH_ERR_ENOMEM_fs_btree_cache_init; + + if (rcu_pending_init(&bc->pending[0], &c->btree_trans_barrier, __bkey_cached_free) || + rcu_pending_init(&bc->pending[1], &c->btree_trans_barrier, __bkey_cached_free)) + return -BCH_ERR_ENOMEM_fs_btree_cache_init; + if (rhashtable_init(&bc->table, &bch2_btree_key_cache_params)) return -BCH_ERR_ENOMEM_fs_btree_cache_init; @@ -757,13 +785,15 @@ void bch2_btree_key_cache_to_text(struct printbuf *out, struct btree_key_cache * prt_printf(out, "keys:\t%lu\r\n", atomic_long_read(&bc->nr_keys)); prt_printf(out, "dirty:\t%lu\r\n", atomic_long_read(&bc->nr_dirty)); prt_printf(out, "table size:\t%u\r\n", bc->table.tbl->size); - - prt_printf(out, "\nshrinker:\n"); + prt_newline(out); + prt_printf(out, "shrinker:\n"); prt_printf(out, "requested_to_free:\t%lu\r\n", bc->requested_to_free); prt_printf(out, "freed:\t%lu\r\n", bc->freed); prt_printf(out, "skipped_dirty:\t%lu\r\n", bc->skipped_dirty); prt_printf(out, "skipped_accessed:\t%lu\r\n", bc->skipped_accessed); prt_printf(out, "skipped_lock_fail:\t%lu\r\n", bc->skipped_lock_fail); + prt_newline(out); + prt_printf(out, "pending:\t%lu\r\n", per_cpu_sum(bc->nr_pending)); } void bch2_btree_key_cache_exit(void) diff --git a/fs/bcachefs/btree_key_cache_types.h b/fs/bcachefs/btree_key_cache_types.h index e026c65f54e1..722f1ed10551 100644 --- a/fs/bcachefs/btree_key_cache_types.h +++ b/fs/bcachefs/btree_key_cache_types.h @@ -2,6 +2,8 @@ #ifndef _BCACHEFS_BTREE_KEY_CACHE_TYPES_H #define _BCACHEFS_BTREE_KEY_CACHE_TYPES_H +#include "rcu_pending.h" + struct btree_key_cache { struct rhashtable table; bool table_init_done; @@ -9,6 +11,10 @@ struct btree_key_cache { struct shrinker *shrink; unsigned shrink_iter; + /* 0: non pcpu reader locks, 1: pcpu reader locks */ + struct rcu_pending pending[2]; + size_t __percpu *nr_pending; + atomic_long_t nr_keys; atomic_long_t nr_dirty; diff --git a/fs/bcachefs/btree_types.h b/fs/bcachefs/btree_types.h index bca56b6359e7..0df07929c545 100644 --- a/fs/bcachefs/btree_types.h +++ b/fs/bcachefs/btree_types.h @@ -395,7 +395,6 @@ struct bkey_cached { u64 seq; struct bkey_i *k; - struct rcu_head rcu; }; From 5396e5af3c1032b659bd6b1c1ec18e8af275a746 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 21 Aug 2024 13:43:03 -0700 Subject: [PATCH 132/302] bcachefs: Fix format specifier in bch2_btree_key_cache_to_text() When building for a 32-bit architecture, for which 'size_t' is 'unsigned int', there is a compiler warning due to use of '%lu': In file included from fs/bcachefs/vstructs.h:5, from fs/bcachefs/bcachefs_format.h:80, from fs/bcachefs/bcachefs.h:207, from fs/bcachefs/btree_key_cache.c:3: fs/bcachefs/btree_key_cache.c: In function 'bch2_btree_key_cache_to_text': fs/bcachefs/btree_key_cache.c:795:25: error: format '%lu' expects argument of type 'long unsigned int', but argument 3 has type 'size_t' {aka 'unsigned int'} [-Werror=format=] 795 | prt_printf(out, "pending:\t%lu\r\n", per_cpu_sum(bc->nr_pending)); | ^~~~~~~~~~~~~~~~~~~ fs/bcachefs/util.h:78:63: note: in definition of macro 'prt_printf' 78 | #define prt_printf(_out, ...) bch2_prt_printf(_out, __VA_ARGS__) | ^~~~~~~~~~~ fs/bcachefs/btree_key_cache.c:795:38: note: format string is defined here 795 | prt_printf(out, "pending:\t%lu\r\n", per_cpu_sum(bc->nr_pending)); | ~~^ | | | long unsigned int | %u cc1: all warnings being treated as errors Use the proper specifier, '%zu', to resolve the warning. Fixes: e447e49977b8 ("bcachefs: key cache can now allocate from pending") Signed-off-by: Nathan Chancellor Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_key_cache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/bcachefs/btree_key_cache.c b/fs/bcachefs/btree_key_cache.c index 3048adde9284..2e49ca71194f 100644 --- a/fs/bcachefs/btree_key_cache.c +++ b/fs/bcachefs/btree_key_cache.c @@ -793,7 +793,7 @@ void bch2_btree_key_cache_to_text(struct printbuf *out, struct btree_key_cache * prt_printf(out, "skipped_accessed:\t%lu\r\n", bc->skipped_accessed); prt_printf(out, "skipped_lock_fail:\t%lu\r\n", bc->skipped_lock_fail); prt_newline(out); - prt_printf(out, "pending:\t%lu\r\n", per_cpu_sum(bc->nr_pending)); + prt_printf(out, "pending:\t%zu\r\n", per_cpu_sum(bc->nr_pending)); } void bch2_btree_key_cache_exit(void) From 8573dd3474e8ec067b3c0c80d591c6de1d94e1e9 Mon Sep 17 00:00:00 2001 From: Thorsten Blum Date: Wed, 21 Aug 2024 18:29:22 +0200 Subject: [PATCH 133/302] bcachefs: Annotate struct bucket_array with __counted_by() Add the __counted_by compiler attribute to the flexible array member bucket to improve access bounds-checking via CONFIG_UBSAN_BOUNDS and CONFIG_FORTIFY_SOURCE. Signed-off-by: Thorsten Blum Signed-off-by: Kent Overstreet --- fs/bcachefs/buckets_types.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/bcachefs/buckets_types.h b/fs/bcachefs/buckets_types.h index c9698cdf866f..a19460a1bb7c 100644 --- a/fs/bcachefs/buckets_types.h +++ b/fs/bcachefs/buckets_types.h @@ -24,7 +24,7 @@ struct bucket_array { u16 first_bucket; size_t nbuckets; size_t nbuckets_minus_first; - struct bucket b[]; + struct bucket b[] __counted_by(nbuckets); }; struct bucket_gens { From afefc986b7d04c67de870558f76a7d0e7293eac5 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Mon, 15 Jul 2024 16:53:49 -0400 Subject: [PATCH 134/302] bcachefs: data_allowed is now an opts.h option need this so cmd_option in userspace can handle it Signed-off-by: Kent Overstreet --- fs/bcachefs/opts.c | 9 +++++++++ fs/bcachefs/opts.h | 6 ++++++ fs/bcachefs/util.c | 2 +- fs/bcachefs/util.h | 2 +- 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/fs/bcachefs/opts.c b/fs/bcachefs/opts.c index e10fc1da71b1..8e2fd064b50f 100644 --- a/fs/bcachefs/opts.c +++ b/fs/bcachefs/opts.c @@ -230,6 +230,8 @@ const struct bch_option bch2_opt_table[] = { #define OPT_STR_NOLIMIT(_choices) .type = BCH_OPT_STR, \ .min = 0, .max = U64_MAX, \ .choices = _choices +#define OPT_BITFIELD(_choices) .type = BCH_OPT_BITFIELD, \ + .choices = _choices #define OPT_FN(_fn) .type = BCH_OPT_FN, .fn = _fn #define x(_name, _bits, _flags, _type, _sb_opt, _default, _hint, _help) \ @@ -376,6 +378,13 @@ int bch2_opt_parse(struct bch_fs *c, *res = ret; break; + case BCH_OPT_BITFIELD: { + s64 v = bch2_read_flag_list(val, opt->choices); + if (v < 0) + return v; + *res = v; + break; + } case BCH_OPT_FN: ret = opt->fn.parse(c, val, res, err); diff --git a/fs/bcachefs/opts.h b/fs/bcachefs/opts.h index cda1725702ea..1c7b68b25bb7 100644 --- a/fs/bcachefs/opts.h +++ b/fs/bcachefs/opts.h @@ -70,6 +70,7 @@ enum opt_type { BCH_OPT_BOOL, BCH_OPT_UINT, BCH_OPT_STR, + BCH_OPT_BITFIELD, BCH_OPT_FN, }; @@ -477,6 +478,11 @@ enum fsck_err_opts { BCH2_NO_SB_OPT, 1, \ "n", "Data written to this device will be considered\n"\ "to have already been replicated n times") \ + x(data_allowed, u8, \ + OPT_DEVICE, \ + OPT_BITFIELD(__bch2_data_types), \ + BCH2_NO_SB_OPT, BIT(BCH_DATA_journal)|BIT(BCH_DATA_btree)|BIT(BCH_DATA_user),\ + "types", "Allowed data types for this device: journal, btree, and/or user")\ x(btree_node_prefetch, u8, \ OPT_FS|OPT_MOUNT|OPT_RUNTIME, \ OPT_BOOL(), \ diff --git a/fs/bcachefs/util.c b/fs/bcachefs/util.c index 144d8df48b1e..8b4e8395a908 100644 --- a/fs/bcachefs/util.c +++ b/fs/bcachefs/util.c @@ -204,7 +204,7 @@ STRTO_H(strtoll, long long) STRTO_H(strtoull, unsigned long long) STRTO_H(strtou64, u64) -u64 bch2_read_flag_list(char *opt, const char * const list[]) +u64 bch2_read_flag_list(const char *opt, const char * const list[]) { u64 ret = 0; char *p, *s, *d = kstrdup(opt, GFP_KERNEL); diff --git a/fs/bcachefs/util.h b/fs/bcachefs/util.h index 902b7f5406a2..fb02c1c36004 100644 --- a/fs/bcachefs/util.h +++ b/fs/bcachefs/util.h @@ -195,7 +195,7 @@ static inline int bch2_strtoul_h(const char *cp, long *res) bool bch2_is_zero(const void *, size_t); -u64 bch2_read_flag_list(char *, const char * const[]); +u64 bch2_read_flag_list(const char *, const char * const[]); void bch2_prt_u64_base2_nbits(struct printbuf *, u64, unsigned); void bch2_prt_u64_base2(struct printbuf *, u64); From 4aedeac5703e06f5e9c1eeee6f77136bcc15afdb Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Mon, 15 Jul 2024 19:26:46 -0400 Subject: [PATCH 135/302] bcachefs: bch2_opt_set_sb() can now set (some) device options Signed-off-by: Kent Overstreet --- fs/bcachefs/opts.c | 42 +++++++++++++++++++++++++++++++++--------- fs/bcachefs/opts.h | 10 ++++++++-- fs/bcachefs/sysfs.c | 11 ++--------- 3 files changed, 43 insertions(+), 20 deletions(-) diff --git a/fs/bcachefs/opts.c b/fs/bcachefs/opts.c index 8e2fd064b50f..2e6e58360789 100644 --- a/fs/bcachefs/opts.c +++ b/fs/bcachefs/opts.c @@ -617,10 +617,20 @@ int bch2_opts_from_sb(struct bch_opts *opts, struct bch_sb *sb) return 0; } -void __bch2_opt_set_sb(struct bch_sb *sb, const struct bch_option *opt, u64 v) +struct bch_dev_sb_opt_set { + void (*set_sb)(struct bch_member *, u64); +}; + +static const struct bch_dev_sb_opt_set bch2_dev_sb_opt_setters [] = { +#define x(n, set) [Opt_##n] = { .set_sb = SET_##set }, + BCH_DEV_OPTS() +#undef x +}; + +void __bch2_opt_set_sb(struct bch_sb *sb, int dev_idx, + const struct bch_option *opt, u64 v) { - if (opt->set_sb == SET_BCH2_NO_SB_OPT) - return; + enum bch_opt_id id = opt - bch2_opt_table; if (opt->flags & OPT_SB_FIELD_SECTORS) v >>= 9; @@ -628,16 +638,30 @@ void __bch2_opt_set_sb(struct bch_sb *sb, const struct bch_option *opt, u64 v) if (opt->flags & OPT_SB_FIELD_ILOG2) v = ilog2(v); - opt->set_sb(sb, v); + if (opt->flags & OPT_FS) { + if (opt->set_sb != SET_BCH2_NO_SB_OPT) + opt->set_sb(sb, v); + } + + if ((opt->flags & OPT_DEVICE) && dev_idx >= 0) { + if (WARN(!bch2_member_exists(sb, dev_idx), + "tried to set device option %s on nonexistent device %i", + opt->attr.name, dev_idx)) + return; + + struct bch_member *m = bch2_members_v2_get_mut(sb, dev_idx); + + const struct bch_dev_sb_opt_set *set = bch2_dev_sb_opt_setters + id; + if (set->set_sb) + set->set_sb(m, v); + } } -void bch2_opt_set_sb(struct bch_fs *c, const struct bch_option *opt, u64 v) +void bch2_opt_set_sb(struct bch_fs *c, struct bch_dev *ca, + const struct bch_option *opt, u64 v) { - if (opt->set_sb == SET_BCH2_NO_SB_OPT) - return; - mutex_lock(&c->sb_lock); - __bch2_opt_set_sb(c->disk_sb.sb, opt, v); + __bch2_opt_set_sb(c->disk_sb.sb, ca ? ca->dev_idx : -1, opt, v); bch2_write_super(c); mutex_unlock(&c->sb_lock); } diff --git a/fs/bcachefs/opts.h b/fs/bcachefs/opts.h index 1c7b68b25bb7..32f895dfb330 100644 --- a/fs/bcachefs/opts.h +++ b/fs/bcachefs/opts.h @@ -490,6 +490,10 @@ enum fsck_err_opts { NULL, "BTREE_ITER_prefetch casuse btree nodes to be\n"\ " prefetched sequentially") +#define BCH_DEV_OPTS() \ + x(discard, BCH_MEMBER_DISCARD) \ + x(data_allowed, BCH_MEMBER_DATA_ALLOWED) + struct bch_opts { #define x(_name, _bits, ...) unsigned _name##_defined:1; BCH_OPTS() @@ -569,8 +573,10 @@ void bch2_opt_set_by_id(struct bch_opts *, enum bch_opt_id, u64); u64 bch2_opt_from_sb(struct bch_sb *, enum bch_opt_id); int bch2_opts_from_sb(struct bch_opts *, struct bch_sb *); -void __bch2_opt_set_sb(struct bch_sb *, const struct bch_option *, u64); -void bch2_opt_set_sb(struct bch_fs *, const struct bch_option *, u64); +void __bch2_opt_set_sb(struct bch_sb *, int, const struct bch_option *, u64); + +struct bch_dev; +void bch2_opt_set_sb(struct bch_fs *, struct bch_dev *, const struct bch_option *, u64); int bch2_opt_lookup(const char *); int bch2_opt_validate(const struct bch_option *, u64, struct printbuf *); diff --git a/fs/bcachefs/sysfs.c b/fs/bcachefs/sysfs.c index abaf9510fd96..d97ac9444592 100644 --- a/fs/bcachefs/sysfs.c +++ b/fs/bcachefs/sysfs.c @@ -674,7 +674,7 @@ STORE(bch2_fs_opts_dir) if (ret < 0) goto err; - bch2_opt_set_sb(c, opt, v); + bch2_opt_set_sb(c, NULL, opt, v); bch2_opt_set_by_id(&c->opts, id, v); if (v && @@ -826,14 +826,7 @@ STORE(bch2_dev) if (attr == &sysfs_discard) { bool v = strtoul_or_return(buf); - mutex_lock(&c->sb_lock); - mi = bch2_members_v2_get_mut(c->disk_sb.sb, ca->dev_idx); - - if (v != BCH_MEMBER_DISCARD(mi)) { - SET_BCH_MEMBER_DISCARD(mi, v); - bch2_write_super(c); - } - mutex_unlock(&c->sb_lock); + bch2_opt_set_sb(c, ca, bch2_opt_table + Opt_discard, v); } if (attr == &sysfs_durability) { From 9092a38a3dfd70326a47eb8d3ff57e995e770ed4 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Mon, 15 Jul 2024 19:54:51 -0400 Subject: [PATCH 136/302] bcachefs: Opt_durability can now be set via bch2_opt_set_sb() Signed-off-by: Kent Overstreet --- fs/bcachefs/opts.c | 7 ++++++- fs/bcachefs/opts.h | 28 +++++++++++++++------------- fs/bcachefs/sysfs.c | 10 +--------- 3 files changed, 22 insertions(+), 23 deletions(-) diff --git a/fs/bcachefs/opts.c b/fs/bcachefs/opts.c index 2e6e58360789..0770aebef6d8 100644 --- a/fs/bcachefs/opts.c +++ b/fs/bcachefs/opts.c @@ -623,7 +623,7 @@ struct bch_dev_sb_opt_set { static const struct bch_dev_sb_opt_set bch2_dev_sb_opt_setters [] = { #define x(n, set) [Opt_##n] = { .set_sb = SET_##set }, - BCH_DEV_OPTS() + BCH_DEV_OPT_SETTERS() #undef x }; @@ -638,6 +638,9 @@ void __bch2_opt_set_sb(struct bch_sb *sb, int dev_idx, if (opt->flags & OPT_SB_FIELD_ILOG2) v = ilog2(v); + if (opt->flags & OPT_SB_FIELD_ONE_BIAS) + v++; + if (opt->flags & OPT_FS) { if (opt->set_sb != SET_BCH2_NO_SB_OPT) opt->set_sb(sb, v); @@ -654,6 +657,8 @@ void __bch2_opt_set_sb(struct bch_sb *sb, int dev_idx, const struct bch_dev_sb_opt_set *set = bch2_dev_sb_opt_setters + id; if (set->set_sb) set->set_sb(m, v); + else + pr_err("option %s cannot be set via opt_set_sb()", opt->attr.name); } } diff --git a/fs/bcachefs/opts.h b/fs/bcachefs/opts.h index 32f895dfb330..0138717e9f77 100644 --- a/fs/bcachefs/opts.h +++ b/fs/bcachefs/opts.h @@ -53,17 +53,18 @@ void SET_BCH2_NO_SB_OPT(struct bch_sb *, u64); /* When can be set: */ enum opt_flags { - OPT_FS = (1 << 0), /* Filesystem option */ - OPT_DEVICE = (1 << 1), /* Device option */ - OPT_INODE = (1 << 2), /* Inode option */ - OPT_FORMAT = (1 << 3), /* May be specified at format time */ - OPT_MOUNT = (1 << 4), /* May be specified at mount time */ - OPT_RUNTIME = (1 << 5), /* May be specified at runtime */ - OPT_HUMAN_READABLE = (1 << 6), - OPT_MUST_BE_POW_2 = (1 << 7), /* Must be power of 2 */ - OPT_SB_FIELD_SECTORS = (1 << 8),/* Superblock field is >> 9 of actual value */ - OPT_SB_FIELD_ILOG2 = (1 << 9), /* Superblock field is ilog2 of actual value */ - OPT_HIDDEN = (1 << 10), + OPT_FS = BIT(0), /* Filesystem option */ + OPT_DEVICE = BIT(1), /* Device option */ + OPT_INODE = BIT(2), /* Inode option */ + OPT_FORMAT = BIT(3), /* May be specified at format time */ + OPT_MOUNT = BIT(4), /* May be specified at mount time */ + OPT_RUNTIME = BIT(5), /* May be specified at runtime */ + OPT_HUMAN_READABLE = BIT(6), + OPT_MUST_BE_POW_2 = BIT(7), /* Must be power of 2 */ + OPT_SB_FIELD_SECTORS = BIT(8), /* Superblock field is >> 9 of actual value */ + OPT_SB_FIELD_ILOG2 = BIT(9), /* Superblock field is ilog2 of actual value */ + OPT_SB_FIELD_ONE_BIAS = BIT(10), /* 0 means default value */ + OPT_HIDDEN = BIT(11), }; enum opt_type { @@ -473,7 +474,7 @@ enum fsck_err_opts { BCH2_NO_SB_OPT, 0, \ "size", "Size of filesystem on device") \ x(durability, u8, \ - OPT_DEVICE, \ + OPT_DEVICE|OPT_SB_FIELD_ONE_BIAS, \ OPT_UINT(0, BCH_REPLICAS_MAX), \ BCH2_NO_SB_OPT, 1, \ "n", "Data written to this device will be considered\n"\ @@ -490,8 +491,9 @@ enum fsck_err_opts { NULL, "BTREE_ITER_prefetch casuse btree nodes to be\n"\ " prefetched sequentially") -#define BCH_DEV_OPTS() \ +#define BCH_DEV_OPT_SETTERS() \ x(discard, BCH_MEMBER_DISCARD) \ + x(durability, BCH_MEMBER_DURABILITY) \ x(data_allowed, BCH_MEMBER_DATA_ALLOWED) struct bch_opts { diff --git a/fs/bcachefs/sysfs.c b/fs/bcachefs/sysfs.c index d97ac9444592..89da5323cf59 100644 --- a/fs/bcachefs/sysfs.c +++ b/fs/bcachefs/sysfs.c @@ -821,7 +821,6 @@ STORE(bch2_dev) { struct bch_dev *ca = container_of(kobj, struct bch_dev, kobj); struct bch_fs *c = ca->fs; - struct bch_member *mi; if (attr == &sysfs_discard) { bool v = strtoul_or_return(buf); @@ -832,14 +831,7 @@ STORE(bch2_dev) if (attr == &sysfs_durability) { u64 v = strtoul_or_return(buf); - mutex_lock(&c->sb_lock); - mi = bch2_members_v2_get_mut(c->disk_sb.sb, ca->dev_idx); - - if (v + 1 != BCH_MEMBER_DURABILITY(mi)) { - SET_BCH_MEMBER_DURABILITY(mi, v + 1); - bch2_write_super(c); - } - mutex_unlock(&c->sb_lock); + bch2_opt_set_sb(c, ca, bch2_opt_table + Opt_durability, v); } if (attr == &sysfs_label) { From 082330c361944a234db42750792b491f3b76d43b Mon Sep 17 00:00:00 2001 From: Youling Tang Date: Tue, 16 Jul 2024 10:58:16 +0800 Subject: [PATCH 137/302] bcachefs: allocate inode by using alloc_inode_sb() The inode allocation is supposed to use alloc_inode_sb(), so convert kmem_cache_alloc() to alloc_inode_sb(). It will also fix [1] to avoid the NULL pointer dereference BUG in list_lru_add() when CONFIG_MEMCG is enabled. Links: [1]: https://lore.kernel.org/all/20589721-46c0-4344-b2ef-6ab48bbe2ea5@linux.dev/ [2]: https://lore.kernel.org/all/7db60e36-9c96-4938-a28d-a9745e287386@linux.dev/ Fixes: 86d81ec5f5f0 ("bcachefs: Mark bch_inode_info as SLAB_ACCOUNT") Signed-off-by: Youling Tang Signed-off-by: Kent Overstreet --- fs/bcachefs/fs.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/bcachefs/fs.c b/fs/bcachefs/fs.c index 2505daaec458..cb71d05a32bc 100644 --- a/fs/bcachefs/fs.c +++ b/fs/bcachefs/fs.c @@ -293,7 +293,8 @@ static struct inode *bch2_alloc_inode(struct super_block *sb) static struct bch_inode_info *__bch2_new_inode(struct bch_fs *c) { - struct bch_inode_info *inode = kmem_cache_alloc(bch2_inode_cache, GFP_NOFS); + struct bch_inode_info *inode = alloc_inode_sb(c->vfs_sb, + bch2_inode_cache, GFP_NOFS); if (!inode) return NULL; From 094c6a9f5cd567e9fe0e9f0616157a7e6369566e Mon Sep 17 00:00:00 2001 From: Youling Tang Date: Wed, 3 Jul 2024 15:09:55 +0800 Subject: [PATCH 138/302] bcachefs: Mark bch_inode_info as SLAB_ACCOUNT After commit 230e9fc28604 ("slab: add SLAB_ACCOUNT flag"), we need to mark the inode cache as SLAB_ACCOUNT, similar to commit 5d097056c9a0 ("kmemcg: account for certain kmem allocations to memcg") Signed-off-by: Youling Tang Signed-off-by: Kent Overstreet --- fs/bcachefs/fs.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/bcachefs/fs.c b/fs/bcachefs/fs.c index cb71d05a32bc..d90b3082a86f 100644 --- a/fs/bcachefs/fs.c +++ b/fs/bcachefs/fs.c @@ -2236,7 +2236,8 @@ int __init bch2_vfs_init(void) { int ret = -ENOMEM; - bch2_inode_cache = KMEM_CACHE(bch_inode_info, SLAB_RECLAIM_ACCOUNT); + bch2_inode_cache = KMEM_CACHE(bch_inode_info, SLAB_RECLAIM_ACCOUNT | + SLAB_ACCOUNT); if (!bch2_inode_cache) goto err; From abbfc4db50e27ab0d7b5315452bc8faa5eeb19df Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 16 Jul 2024 17:23:10 -0400 Subject: [PATCH 139/302] bcachefs: Add check for btree_path ref overflow Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_iter.c | 24 ++++++++++++------------ fs/bcachefs/btree_iter.h | 26 ++++++++++++++++++-------- fs/bcachefs/btree_update.c | 2 +- 3 files changed, 31 insertions(+), 21 deletions(-) diff --git a/fs/bcachefs/btree_iter.c b/fs/bcachefs/btree_iter.c index 2e84d22e17bd..95afa35833bb 100644 --- a/fs/bcachefs/btree_iter.c +++ b/fs/bcachefs/btree_iter.c @@ -1010,9 +1010,9 @@ retry_all: * the same position: */ if (trans->paths[idx].uptodate) { - __btree_path_get(&trans->paths[idx], false); + __btree_path_get(trans, &trans->paths[idx], false); ret = bch2_btree_path_traverse_one(trans, idx, 0, _THIS_IP_); - __btree_path_put(&trans->paths[idx], false); + __btree_path_put(trans, &trans->paths[idx], false); if (bch2_err_matches(ret, BCH_ERR_transaction_restart) || bch2_err_matches(ret, ENOMEM)) @@ -1225,7 +1225,7 @@ static btree_path_idx_t btree_path_clone(struct btree_trans *trans, btree_path_i { btree_path_idx_t new = btree_path_alloc(trans, src); btree_path_copy(trans, trans->paths + new, trans->paths + src); - __btree_path_get(trans->paths + new, intent); + __btree_path_get(trans, trans->paths + new, intent); #ifdef TRACK_PATH_ALLOCATED trans->paths[new].ip_allocated = ip; #endif @@ -1236,7 +1236,7 @@ __flatten btree_path_idx_t __bch2_btree_path_make_mut(struct btree_trans *trans, btree_path_idx_t path, bool intent, unsigned long ip) { - __btree_path_put(trans->paths + path, intent); + __btree_path_put(trans, trans->paths + path, intent); path = btree_path_clone(trans, path, intent, ip); trans->paths[path].preserve = false; return path; @@ -1361,7 +1361,7 @@ void bch2_path_put(struct btree_trans *trans, btree_path_idx_t path_idx, bool in { struct btree_path *path = trans->paths + path_idx, *dup; - if (!__btree_path_put(path, intent)) + if (!__btree_path_put(trans, path, intent)) return; dup = path->preserve @@ -1392,7 +1392,7 @@ void bch2_path_put(struct btree_trans *trans, btree_path_idx_t path_idx, bool in static void bch2_path_put_nokeep(struct btree_trans *trans, btree_path_idx_t path, bool intent) { - if (!__btree_path_put(trans->paths + path, intent)) + if (!__btree_path_put(trans, trans->paths + path, intent)) return; __bch2_path_free(trans, path); @@ -1716,14 +1716,14 @@ btree_path_idx_t bch2_path_get(struct btree_trans *trans, trans->paths[path_pos].cached == cached && trans->paths[path_pos].btree_id == btree_id && trans->paths[path_pos].level == level) { - __btree_path_get(trans->paths + path_pos, intent); + __btree_path_get(trans, trans->paths + path_pos, intent); path_idx = bch2_btree_path_set_pos(trans, path_pos, pos, intent, ip); path = trans->paths + path_idx; } else { path_idx = btree_path_alloc(trans, path_pos); path = trans->paths + path_idx; - __btree_path_get(path, intent); + __btree_path_get(trans, path, intent); path->pos = pos; path->btree_id = btree_id; path->cached = cached; @@ -2326,7 +2326,7 @@ struct bkey_s_c bch2_btree_iter_peek_upto(struct btree_iter *iter, struct bpos e * advance, same as on exit for iter->path, but only up * to snapshot */ - __btree_path_get(trans->paths + iter->path, iter->flags & BTREE_ITER_intent); + __btree_path_get(trans, trans->paths + iter->path, iter->flags & BTREE_ITER_intent); iter->update_path = iter->path; iter->update_path = bch2_btree_path_set_pos(trans, @@ -2911,9 +2911,9 @@ void bch2_trans_copy_iter(struct btree_iter *dst, struct btree_iter *src) dst->ip_allocated = _RET_IP_; #endif if (src->path) - __btree_path_get(trans->paths + src->path, src->flags & BTREE_ITER_intent); + __btree_path_get(trans, trans->paths + src->path, src->flags & BTREE_ITER_intent); if (src->update_path) - __btree_path_get(trans->paths + src->update_path, src->flags & BTREE_ITER_intent); + __btree_path_get(trans, trans->paths + src->update_path, src->flags & BTREE_ITER_intent); dst->key_cache_path = 0; } @@ -3237,7 +3237,7 @@ void bch2_trans_put(struct btree_trans *trans) bch2_trans_unlock(trans); trans_for_each_update(trans, i) - __btree_path_put(trans->paths + i->path, true); + __btree_path_put(trans, trans->paths + i->path, true); trans->nr_updates = 0; check_btree_paths_leaked(trans); diff --git a/fs/bcachefs/btree_iter.h b/fs/bcachefs/btree_iter.h index 222b7ce8a901..928993118983 100644 --- a/fs/bcachefs/btree_iter.h +++ b/fs/bcachefs/btree_iter.h @@ -6,6 +6,12 @@ #include "btree_types.h" #include "trace.h" +void bch2_trans_updates_to_text(struct printbuf *, struct btree_trans *); +void bch2_btree_path_to_text(struct printbuf *, struct btree_trans *, btree_path_idx_t); +void bch2_trans_paths_to_text(struct printbuf *, struct btree_trans *); +void bch2_dump_trans_updates(struct btree_trans *); +void bch2_dump_trans_paths_updates(struct btree_trans *); + static inline int __bkey_err(const struct bkey *k) { return PTR_ERR_OR_ZERO(k); @@ -13,16 +19,26 @@ static inline int __bkey_err(const struct bkey *k) #define bkey_err(_k) __bkey_err((_k).k) -static inline void __btree_path_get(struct btree_path *path, bool intent) +static inline void __btree_path_get(struct btree_trans *trans, struct btree_path *path, bool intent) { + unsigned idx = path - trans->paths; + + EBUG_ON(!test_bit(idx, trans->paths_allocated)); + if (unlikely(path->ref == U8_MAX)) { + bch2_dump_trans_paths_updates(trans); + panic("path %u refcount overflow\n", idx); + } + path->ref++; path->intent_ref += intent; } -static inline bool __btree_path_put(struct btree_path *path, bool intent) +static inline bool __btree_path_put(struct btree_trans *trans, struct btree_path *path, bool intent) { + EBUG_ON(!test_bit(path - trans->paths, trans->paths_allocated)); EBUG_ON(!path->ref); EBUG_ON(!path->intent_ref && intent); + path->intent_ref -= intent; return --path->ref == 0; } @@ -894,12 +910,6 @@ __bch2_btree_iter_peek_and_restart(struct btree_trans *trans, _ret; \ }) -void bch2_trans_updates_to_text(struct printbuf *, struct btree_trans *); -void bch2_btree_path_to_text(struct printbuf *, struct btree_trans *, btree_path_idx_t); -void bch2_trans_paths_to_text(struct printbuf *, struct btree_trans *); -void bch2_dump_trans_updates(struct btree_trans *); -void bch2_dump_trans_paths_updates(struct btree_trans *); - struct btree_trans *__bch2_trans_get(struct bch_fs *, unsigned); void bch2_trans_put(struct btree_trans *); diff --git a/fs/bcachefs/btree_update.c b/fs/bcachefs/btree_update.c index d6f6df10dcc3..57610c7465b3 100644 --- a/fs/bcachefs/btree_update.c +++ b/fs/bcachefs/btree_update.c @@ -449,7 +449,7 @@ bch2_trans_update_by_path(struct btree_trans *trans, btree_path_idx_t path_idx, } } - __btree_path_get(trans->paths + i->path, true); + __btree_path_get(trans, trans->paths + i->path, true); /* * If a key is present in the key cache, it must also exist in the From 32ed4a620c5405be73a59c6407bd2c6dae9d50c3 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 10 Aug 2022 19:57:46 -0400 Subject: [PATCH 140/302] bcachefs: Btree path tracepoints Fastpath tracepoints, rarely needed, only enabled with CONFIG_BCACHEFS_PATH_TRACEPOINTS. Signed-off-by: Kent Overstreet --- fs/bcachefs/Kconfig | 7 + fs/bcachefs/btree_iter.c | 38 ++- fs/bcachefs/btree_iter.h | 2 + fs/bcachefs/btree_locking.h | 6 +- fs/bcachefs/btree_update.c | 10 +- fs/bcachefs/btree_update_interior.c | 2 +- fs/bcachefs/trace.h | 465 +++++++++++++++++++++++++++- 7 files changed, 508 insertions(+), 22 deletions(-) diff --git a/fs/bcachefs/Kconfig b/fs/bcachefs/Kconfig index 5cdfef3b551a..5bac803ea367 100644 --- a/fs/bcachefs/Kconfig +++ b/fs/bcachefs/Kconfig @@ -87,6 +87,13 @@ config BCACHEFS_SIX_OPTIMISTIC_SPIN is held by another thread, spin for a short while, as long as the thread owning the lock is running. +config BCACHEFS_PATH_TRACEPOINTS + bool "Extra btree_path tracepoints" + depends on BCACHEFS_FS + help + Enable extra tracepoints for debugging btree_path operations; we don't + normally want these enabled because they happen at very high rates. + config MEAN_AND_VARIANCE_UNIT_TEST tristate "mean_and_variance unit tests" if !KUNIT_ALL_TESTS depends on KUNIT diff --git a/fs/bcachefs/btree_iter.c b/fs/bcachefs/btree_iter.c index 95afa35833bb..c71efbdb3f07 100644 --- a/fs/bcachefs/btree_iter.c +++ b/fs/bcachefs/btree_iter.c @@ -1131,6 +1131,8 @@ int bch2_btree_path_traverse_one(struct btree_trans *trans, if (unlikely(!trans->srcu_held)) bch2_trans_srcu_lock(trans); + trace_btree_path_traverse_start(trans, path); + /* * Ensure we obey path->should_be_locked: if it's set, we can't unlock * and re-traverse the path without a transaction restart: @@ -1194,6 +1196,7 @@ int bch2_btree_path_traverse_one(struct btree_trans *trans, out_uptodate: path->uptodate = BTREE_ITER_UPTODATE; + trace_btree_path_traverse_end(trans, path); out: if (bch2_err_matches(ret, BCH_ERR_transaction_restart) != !!trans->restarted) panic("ret %s (%i) trans->restarted %s (%i)\n", @@ -1236,8 +1239,10 @@ __flatten btree_path_idx_t __bch2_btree_path_make_mut(struct btree_trans *trans, btree_path_idx_t path, bool intent, unsigned long ip) { + struct btree_path *old = trans->paths + path; __btree_path_put(trans, trans->paths + path, intent); path = btree_path_clone(trans, path, intent, ip); + trace_btree_path_clone(trans, old, trans->paths + path); trans->paths[path].preserve = false; return path; } @@ -1252,6 +1257,8 @@ __bch2_btree_path_set_pos(struct btree_trans *trans, bch2_trans_verify_not_in_restart(trans); EBUG_ON(!trans->paths[path_idx].ref); + trace_btree_path_set_pos(trans, trans->paths + path_idx, &new_pos); + path_idx = bch2_btree_path_make_mut(trans, path_idx, intent, ip); struct btree_path *path = trans->paths + path_idx; @@ -1368,6 +1375,8 @@ void bch2_path_put(struct btree_trans *trans, btree_path_idx_t path_idx, bool in ? have_path_at_pos(trans, path) : have_node_at_pos(trans, path); + trace_btree_path_free(trans, path_idx, dup); + if (!dup && !(!path->preserve && !is_btree_node(path, path->level))) return; @@ -1421,8 +1430,8 @@ void __noreturn bch2_trans_unlocked_error(struct btree_trans *trans) noinline __cold void bch2_trans_updates_to_text(struct printbuf *buf, struct btree_trans *trans) { - prt_printf(buf, "transaction updates for %s journal seq %llu\n", - trans->fn, trans->journal_res.seq); + prt_printf(buf, "%u transaction updates for %s journal seq %llu\n", + trans->nr_updates, trans->fn, trans->journal_res.seq); printbuf_indent_add(buf, 2); trans_for_each_update(trans, i) { @@ -1464,7 +1473,7 @@ static void bch2_btree_path_to_text_short(struct printbuf *out, struct btree_tra { struct btree_path *path = trans->paths + path_idx; - prt_printf(out, "path: idx %2u ref %u:%u %c %c %c btree=%s l=%u pos ", + prt_printf(out, "path: idx %3u ref %u:%u %c %c %c btree=%s l=%u pos ", path_idx, path->ref, path->intent_ref, path->preserve ? 'P' : ' ', path->should_be_locked ? 'S' : ' ', @@ -1716,6 +1725,8 @@ btree_path_idx_t bch2_path_get(struct btree_trans *trans, trans->paths[path_pos].cached == cached && trans->paths[path_pos].btree_id == btree_id && trans->paths[path_pos].level == level) { + trace_btree_path_get(trans, trans->paths + path_pos, &pos); + __btree_path_get(trans, trans->paths + path_pos, intent); path_idx = bch2_btree_path_set_pos(trans, path_pos, pos, intent, ip); path = trans->paths + path_idx; @@ -1738,6 +1749,8 @@ btree_path_idx_t bch2_path_get(struct btree_trans *trans, path->ip_allocated = ip; #endif trans->paths_sorted = false; + + trace_btree_path_alloc(trans, path); } if (!(flags & BTREE_ITER_nopreserve)) @@ -1857,7 +1870,7 @@ bch2_btree_iter_traverse(struct btree_iter *iter) struct btree_path *path = btree_iter_path(trans, iter); if (btree_path_node(path, path->level)) - btree_path_set_should_be_locked(path); + btree_path_set_should_be_locked(trans, path); return 0; } @@ -1889,7 +1902,7 @@ struct btree *bch2_btree_iter_peek_node(struct btree_iter *iter) iter->path = bch2_btree_path_set_pos(trans, iter->path, b->key.k.p, iter->flags & BTREE_ITER_intent, btree_iter_ip_allocated(iter)); - btree_path_set_should_be_locked(btree_iter_path(trans, iter)); + btree_path_set_should_be_locked(trans, btree_iter_path(trans, iter)); out: bch2_btree_iter_verify_entry_exit(iter); bch2_btree_iter_verify(iter); @@ -1983,7 +1996,7 @@ struct btree *bch2_btree_iter_next_node(struct btree_iter *iter) iter->path = bch2_btree_path_set_pos(trans, iter->path, b->key.k.p, iter->flags & BTREE_ITER_intent, btree_iter_ip_allocated(iter)); - btree_path_set_should_be_locked(btree_iter_path(trans, iter)); + btree_path_set_should_be_locked(trans, btree_iter_path(trans, iter)); EBUG_ON(btree_iter_path(trans, iter)->uptodate); out: bch2_btree_iter_verify_entry_exit(iter); @@ -2155,7 +2168,7 @@ struct bkey_s_c btree_trans_peek_key_cache(struct btree_iter *iter, struct bpos if (unlikely(ret)) return bkey_s_c_err(ret); - btree_path_set_should_be_locked(trans->paths + iter->key_cache_path); + btree_path_set_should_be_locked(trans, trans->paths + iter->key_cache_path); k = bch2_btree_path_peek_slot(trans->paths + iter->key_cache_path, &u); if (k.k && !bkey_err(k)) { @@ -2199,7 +2212,7 @@ static struct bkey_s_c __bch2_btree_iter_peek(struct btree_iter *iter, struct bp goto out; } - btree_path_set_should_be_locked(path); + btree_path_set_should_be_locked(trans, path); k = btree_path_level_peek_all(trans->c, l, &iter->k); @@ -2382,14 +2395,14 @@ struct bkey_s_c bch2_btree_iter_peek_upto(struct btree_iter *iter, struct bpos e iter->flags & BTREE_ITER_intent, btree_iter_ip_allocated(iter)); - btree_path_set_should_be_locked(btree_iter_path(trans, iter)); + btree_path_set_should_be_locked(trans, btree_iter_path(trans, iter)); out_no_locked: if (iter->update_path) { ret = bch2_btree_path_relock(trans, trans->paths + iter->update_path, _THIS_IP_); if (unlikely(ret)) k = bkey_s_c_err(ret); else - btree_path_set_should_be_locked(trans->paths + iter->update_path); + btree_path_set_should_be_locked(trans, trans->paths + iter->update_path); } if (!(iter->flags & BTREE_ITER_all_snapshots)) @@ -2511,6 +2524,7 @@ struct bkey_s_c bch2_btree_iter_peek_prev(struct btree_iter *iter) iter->flags & BTREE_ITER_intent, _THIS_IP_); path = btree_iter_path(trans, iter); + trace_btree_path_save_pos(trans, path, trans->paths + saved_path); saved_k = *k.k; saved_v = k.v; } @@ -2527,7 +2541,7 @@ got_key: continue; } - btree_path_set_should_be_locked(path); + btree_path_set_should_be_locked(trans, path); break; } else if (likely(!bpos_eq(path->l[0].b->data->min_key, POS_MIN))) { /* Advance to previous leaf node: */ @@ -2685,7 +2699,7 @@ struct bkey_s_c bch2_btree_iter_peek_slot(struct btree_iter *iter) } } out: - btree_path_set_should_be_locked(btree_iter_path(trans, iter)); + btree_path_set_should_be_locked(trans, btree_iter_path(trans, iter)); out_no_locked: bch2_btree_iter_verify_entry_exit(iter); bch2_btree_iter_verify(iter); diff --git a/fs/bcachefs/btree_iter.h b/fs/bcachefs/btree_iter.h index 928993118983..172507b4dead 100644 --- a/fs/bcachefs/btree_iter.h +++ b/fs/bcachefs/btree_iter.h @@ -31,6 +31,7 @@ static inline void __btree_path_get(struct btree_trans *trans, struct btree_path path->ref++; path->intent_ref += intent; + trace_btree_path_get_ll(trans, path); } static inline bool __btree_path_put(struct btree_trans *trans, struct btree_path *path, bool intent) @@ -39,6 +40,7 @@ static inline bool __btree_path_put(struct btree_trans *trans, struct btree_path EBUG_ON(!path->ref); EBUG_ON(!path->intent_ref && intent); + trace_btree_path_put_ll(trans, path); path->intent_ref -= intent; return --path->ref == 0; } diff --git a/fs/bcachefs/btree_locking.h b/fs/bcachefs/btree_locking.h index 11a64ead8685..dd0a2a1aa2bc 100644 --- a/fs/bcachefs/btree_locking.h +++ b/fs/bcachefs/btree_locking.h @@ -228,6 +228,9 @@ static inline int __btree_node_lock_nopath(struct btree_trans *trans, bch2_six_check_for_deadlock, trans, ip); WRITE_ONCE(trans->locking, NULL); WRITE_ONCE(trans->locking_wait.start_time, 0); + + if (!ret) + trace_btree_path_lock(trans, _THIS_IP_, b); return ret; } @@ -400,12 +403,13 @@ static inline int bch2_btree_path_upgrade(struct btree_trans *trans, /* misc: */ -static inline void btree_path_set_should_be_locked(struct btree_path *path) +static inline void btree_path_set_should_be_locked(struct btree_trans *trans, struct btree_path *path) { EBUG_ON(!btree_node_locked(path, path->level)); EBUG_ON(path->uptodate); path->should_be_locked = true; + trace_btree_path_should_be_locked(trans, path); } static inline void __btree_path_set_level_up(struct btree_trans *trans, diff --git a/fs/bcachefs/btree_update.c b/fs/bcachefs/btree_update.c index 57610c7465b3..514df618548e 100644 --- a/fs/bcachefs/btree_update.c +++ b/fs/bcachefs/btree_update.c @@ -374,7 +374,7 @@ static noinline int flush_new_cached_update(struct btree_trans *trans, i->key_cache_already_flushed = true; i->flags |= BTREE_TRIGGER_norun; - btree_path_set_should_be_locked(btree_path); + btree_path_set_should_be_locked(trans, btree_path); ret = bch2_trans_update_by_path(trans, path_idx, i->k, flags, ip); out: bch2_path_put(trans, path_idx, true); @@ -422,7 +422,9 @@ bch2_trans_update_by_path(struct btree_trans *trans, btree_path_idx_t path_idx, break; } - if (!cmp && i < trans->updates + trans->nr_updates) { + bool overwrite = !cmp && i < trans->updates + trans->nr_updates; + + if (overwrite) { EBUG_ON(i->insert_trigger_run || i->overwrite_trigger_run); bch2_path_put(trans, i->path, true); @@ -451,6 +453,8 @@ bch2_trans_update_by_path(struct btree_trans *trans, btree_path_idx_t path_idx, __btree_path_get(trans, trans->paths + i->path, true); + trace_update_by_path(trans, path, i, overwrite); + /* * If a key is present in the key cache, it must also exist in the * btree - this is necessary for cache coherency. When iterating over @@ -498,7 +502,7 @@ static noinline int bch2_trans_update_get_key_cache(struct btree_trans *trans, return btree_trans_restart(trans, BCH_ERR_transaction_restart_key_cache_raced); } - btree_path_set_should_be_locked(trans->paths + iter->key_cache_path); + btree_path_set_should_be_locked(trans, trans->paths + iter->key_cache_path); } return 0; diff --git a/fs/bcachefs/btree_update_interior.c b/fs/bcachefs/btree_update_interior.c index 8fd112026e7a..396d052474bb 100644 --- a/fs/bcachefs/btree_update_interior.c +++ b/fs/bcachefs/btree_update_interior.c @@ -1981,7 +1981,7 @@ int __bch2_foreground_maybe_merge(struct btree_trans *trans, if (ret) goto err; - btree_path_set_should_be_locked(trans->paths + sib_path); + btree_path_set_should_be_locked(trans, trans->paths + sib_path); m = trans->paths[sib_path].l[level].b; diff --git a/fs/bcachefs/trace.h b/fs/bcachefs/trace.h index c62f00322d1e..5597b9d6297f 100644 --- a/fs/bcachefs/trace.h +++ b/fs/bcachefs/trace.h @@ -3,7 +3,6 @@ #define TRACE_SYSTEM bcachefs #if !defined(_TRACE_BCACHEFS_H) || defined(TRACE_HEADER_MULTI_READ) -#define _TRACE_BCACHEFS_H #include @@ -558,6 +557,7 @@ TRACE_EVENT(btree_path_relock_fail, __field(unsigned long, caller_ip ) __field(u8, btree_id ) __field(u8, level ) + __field(u8, path_idx) TRACE_BPOS_entries(pos) __array(char, node, 24 ) __field(u8, self_read_count ) @@ -575,7 +575,8 @@ TRACE_EVENT(btree_path_relock_fail, strscpy(__entry->trans_fn, trans->fn, sizeof(__entry->trans_fn)); __entry->caller_ip = caller_ip; __entry->btree_id = path->btree_id; - __entry->level = path->level; + __entry->level = level; + __entry->path_idx = path - trans->paths; TRACE_BPOS_assign(pos, path->pos); c = bch2_btree_node_lock_counts(trans, NULL, &path->l[level].b->c, level); @@ -588,7 +589,7 @@ TRACE_EVENT(btree_path_relock_fail, c = six_lock_counts(&path->l[level].b->c.lock); __entry->read_count = c.n[SIX_LOCK_read]; __entry->intent_count = c.n[SIX_LOCK_intent]; - scnprintf(__entry->node, sizeof(__entry->node), "%px", b); + scnprintf(__entry->node, sizeof(__entry->node), "%px", &b->c); } __entry->iter_lock_seq = path->l[level].lock_seq; __entry->node_lock_seq = is_btree_node(path, level) @@ -596,9 +597,10 @@ TRACE_EVENT(btree_path_relock_fail, : 0; ), - TP_printk("%s %pS btree %s pos %llu:%llu:%u level %u node %s held %u:%u lock count %u:%u iter seq %u lock seq %u", + TP_printk("%s %pS\nidx %2u btree %s pos %llu:%llu:%u level %u node %s held %u:%u lock count %u:%u iter seq %u lock seq %u", __entry->trans_fn, (void *) __entry->caller_ip, + __entry->path_idx, bch2_btree_id_str(__entry->btree_id), __entry->pos_inode, __entry->pos_offset, @@ -625,6 +627,7 @@ TRACE_EVENT(btree_path_upgrade_fail, __field(unsigned long, caller_ip ) __field(u8, btree_id ) __field(u8, level ) + __field(u8, path_idx) TRACE_BPOS_entries(pos) __field(u8, locked ) __field(u8, self_read_count ) @@ -642,6 +645,7 @@ TRACE_EVENT(btree_path_upgrade_fail, __entry->caller_ip = caller_ip; __entry->btree_id = path->btree_id; __entry->level = level; + __entry->path_idx = path - trans->paths; TRACE_BPOS_assign(pos, path->pos); __entry->locked = btree_node_locked(path, level); @@ -657,9 +661,10 @@ TRACE_EVENT(btree_path_upgrade_fail, : 0; ), - TP_printk("%s %pS btree %s pos %llu:%llu:%u level %u locked %u held %u:%u lock count %u:%u iter seq %u lock seq %u", + TP_printk("%s %pS\nidx %2u btree %s pos %llu:%llu:%u level %u locked %u held %u:%u lock count %u:%u iter seq %u lock seq %u", __entry->trans_fn, (void *) __entry->caller_ip, + __entry->path_idx, bch2_btree_id_str(__entry->btree_id), __entry->pos_inode, __entry->pos_offset, @@ -1438,6 +1443,456 @@ TRACE_EVENT(error_downcast, TP_printk("%s -> %s %s", __entry->bch_err, __entry->std_err, __entry->ip) ); +#ifdef CONFIG_BCACHEFS_PATH_TRACEPOINTS + +TRACE_EVENT(update_by_path, + TP_PROTO(struct btree_trans *trans, struct btree_path *path, + struct btree_insert_entry *i, bool overwrite), + TP_ARGS(trans, path, i, overwrite), + + TP_STRUCT__entry( + __array(char, trans_fn, 32 ) + __field(btree_path_idx_t, path_idx ) + __field(u8, btree_id ) + TRACE_BPOS_entries(pos) + __field(u8, overwrite ) + __field(btree_path_idx_t, update_idx ) + __field(btree_path_idx_t, nr_updates ) + ), + + TP_fast_assign( + strscpy(__entry->trans_fn, trans->fn, sizeof(__entry->trans_fn)); + __entry->path_idx = path - trans->paths; + __entry->btree_id = path->btree_id; + TRACE_BPOS_assign(pos, path->pos); + __entry->overwrite = overwrite; + __entry->update_idx = i - trans->updates; + __entry->nr_updates = trans->nr_updates; + ), + + TP_printk("%s path %3u btree %s pos %llu:%llu:%u overwrite %u update %u/%u", + __entry->trans_fn, + __entry->path_idx, + bch2_btree_id_str(__entry->btree_id), + __entry->pos_inode, + __entry->pos_offset, + __entry->pos_snapshot, + __entry->overwrite, + __entry->update_idx, + __entry->nr_updates) +); + +TRACE_EVENT(btree_path_lock, + TP_PROTO(struct btree_trans *trans, + unsigned long caller_ip, + struct btree_bkey_cached_common *b), + TP_ARGS(trans, caller_ip, b), + + TP_STRUCT__entry( + __array(char, trans_fn, 32 ) + __field(unsigned long, caller_ip ) + __field(u8, btree_id ) + __field(u8, level ) + __array(char, node, 24 ) + __field(u32, lock_seq ) + ), + + TP_fast_assign( + strscpy(__entry->trans_fn, trans->fn, sizeof(__entry->trans_fn)); + __entry->caller_ip = caller_ip; + __entry->btree_id = b->btree_id; + __entry->level = b->level; + + scnprintf(__entry->node, sizeof(__entry->node), "%px", b); + __entry->lock_seq = six_lock_seq(&b->lock); + ), + + TP_printk("%s %pS\nbtree %s level %u node %s lock seq %u", + __entry->trans_fn, + (void *) __entry->caller_ip, + bch2_btree_id_str(__entry->btree_id), + __entry->level, + __entry->node, + __entry->lock_seq) +); + +DECLARE_EVENT_CLASS(btree_path_ev, + TP_PROTO(struct btree_trans *trans, struct btree_path *path), + TP_ARGS(trans, path), + + TP_STRUCT__entry( + __field(u16, idx ) + __field(u8, ref ) + __field(u8, btree_id ) + TRACE_BPOS_entries(pos) + ), + + TP_fast_assign( + __entry->idx = path - trans->paths; + __entry->ref = path->ref; + __entry->btree_id = path->btree_id; + TRACE_BPOS_assign(pos, path->pos); + ), + + TP_printk("path %3u ref %u btree %s pos %llu:%llu:%u", + __entry->idx, __entry->ref, + bch2_btree_id_str(__entry->btree_id), + __entry->pos_inode, + __entry->pos_offset, + __entry->pos_snapshot) +); + +DEFINE_EVENT(btree_path_ev, btree_path_get_ll, + TP_PROTO(struct btree_trans *trans, struct btree_path *path), + TP_ARGS(trans, path) +); + +DEFINE_EVENT(btree_path_ev, btree_path_put_ll, + TP_PROTO(struct btree_trans *trans, struct btree_path *path), + TP_ARGS(trans, path) +); + +DEFINE_EVENT(btree_path_ev, btree_path_should_be_locked, + TP_PROTO(struct btree_trans *trans, struct btree_path *path), + TP_ARGS(trans, path) +); + +TRACE_EVENT(btree_path_alloc, + TP_PROTO(struct btree_trans *trans, struct btree_path *path), + TP_ARGS(trans, path), + + TP_STRUCT__entry( + __field(btree_path_idx_t, idx ) + __field(u8, locks_want ) + __field(u8, btree_id ) + TRACE_BPOS_entries(pos) + ), + + TP_fast_assign( + __entry->idx = path - trans->paths; + __entry->locks_want = path->locks_want; + __entry->btree_id = path->btree_id; + TRACE_BPOS_assign(pos, path->pos); + ), + + TP_printk("path %3u btree %s locks_want %u pos %llu:%llu:%u", + __entry->idx, + bch2_btree_id_str(__entry->btree_id), + __entry->locks_want, + __entry->pos_inode, + __entry->pos_offset, + __entry->pos_snapshot) +); + +TRACE_EVENT(btree_path_get, + TP_PROTO(struct btree_trans *trans, struct btree_path *path, struct bpos *new_pos), + TP_ARGS(trans, path, new_pos), + + TP_STRUCT__entry( + __field(btree_path_idx_t, idx ) + __field(u8, ref ) + __field(u8, preserve ) + __field(u8, locks_want ) + __field(u8, btree_id ) + TRACE_BPOS_entries(old_pos) + TRACE_BPOS_entries(new_pos) + ), + + TP_fast_assign( + __entry->idx = path - trans->paths; + __entry->ref = path->ref; + __entry->preserve = path->preserve; + __entry->locks_want = path->locks_want; + __entry->btree_id = path->btree_id; + TRACE_BPOS_assign(old_pos, path->pos); + TRACE_BPOS_assign(new_pos, *new_pos); + ), + + TP_printk(" path %3u ref %u preserve %u btree %s locks_want %u pos %llu:%llu:%u -> %llu:%llu:%u", + __entry->idx, + __entry->ref, + __entry->preserve, + bch2_btree_id_str(__entry->btree_id), + __entry->locks_want, + __entry->old_pos_inode, + __entry->old_pos_offset, + __entry->old_pos_snapshot, + __entry->new_pos_inode, + __entry->new_pos_offset, + __entry->new_pos_snapshot) +); + +DECLARE_EVENT_CLASS(btree_path_clone, + TP_PROTO(struct btree_trans *trans, struct btree_path *path, struct btree_path *new), + TP_ARGS(trans, path, new), + + TP_STRUCT__entry( + __field(btree_path_idx_t, idx ) + __field(u8, new_idx ) + __field(u8, btree_id ) + __field(u8, ref ) + __field(u8, preserve ) + TRACE_BPOS_entries(pos) + ), + + TP_fast_assign( + __entry->idx = path - trans->paths; + __entry->new_idx = new - trans->paths; + __entry->btree_id = path->btree_id; + __entry->ref = path->ref; + __entry->preserve = path->preserve; + TRACE_BPOS_assign(pos, path->pos); + ), + + TP_printk(" path %3u ref %u preserve %u btree %s %llu:%llu:%u -> %u", + __entry->idx, + __entry->ref, + __entry->preserve, + bch2_btree_id_str(__entry->btree_id), + __entry->pos_inode, + __entry->pos_offset, + __entry->pos_snapshot, + __entry->new_idx) +); + +DEFINE_EVENT(btree_path_clone, btree_path_clone, + TP_PROTO(struct btree_trans *trans, struct btree_path *path, struct btree_path *new), + TP_ARGS(trans, path, new) +); + +DEFINE_EVENT(btree_path_clone, btree_path_save_pos, + TP_PROTO(struct btree_trans *trans, struct btree_path *path, struct btree_path *new), + TP_ARGS(trans, path, new) +); + +DECLARE_EVENT_CLASS(btree_path_traverse, + TP_PROTO(struct btree_trans *trans, + struct btree_path *path), + TP_ARGS(trans, path), + + TP_STRUCT__entry( + __array(char, trans_fn, 32 ) + __field(btree_path_idx_t, idx ) + __field(u8, ref ) + __field(u8, preserve ) + __field(u8, should_be_locked ) + __field(u8, btree_id ) + __field(u8, level ) + TRACE_BPOS_entries(pos) + __field(u8, locks_want ) + __field(u8, nodes_locked ) + __array(char, node0, 24 ) + __array(char, node1, 24 ) + __array(char, node2, 24 ) + __array(char, node3, 24 ) + ), + + TP_fast_assign( + strscpy(__entry->trans_fn, trans->fn, sizeof(__entry->trans_fn)); + + __entry->idx = path - trans->paths; + __entry->ref = path->ref; + __entry->preserve = path->preserve; + __entry->btree_id = path->btree_id; + __entry->level = path->level; + TRACE_BPOS_assign(pos, path->pos); + + __entry->locks_want = path->locks_want; + __entry->nodes_locked = path->nodes_locked; + struct btree *b = path->l[0].b; + if (IS_ERR(b)) + strscpy(__entry->node0, bch2_err_str(PTR_ERR(b)), sizeof(__entry->node0)); + else + scnprintf(__entry->node0, sizeof(__entry->node0), "%px", &b->c); + b = path->l[1].b; + if (IS_ERR(b)) + strscpy(__entry->node1, bch2_err_str(PTR_ERR(b)), sizeof(__entry->node0)); + else + scnprintf(__entry->node1, sizeof(__entry->node0), "%px", &b->c); + b = path->l[2].b; + if (IS_ERR(b)) + strscpy(__entry->node2, bch2_err_str(PTR_ERR(b)), sizeof(__entry->node0)); + else + scnprintf(__entry->node2, sizeof(__entry->node0), "%px", &b->c); + b = path->l[3].b; + if (IS_ERR(b)) + strscpy(__entry->node3, bch2_err_str(PTR_ERR(b)), sizeof(__entry->node0)); + else + scnprintf(__entry->node3, sizeof(__entry->node0), "%px", &b->c); + ), + + TP_printk("%s\npath %3u ref %u preserve %u btree %s %llu:%llu:%u level %u locks_want %u\n" + "locks %u %u %u %u node %s %s %s %s", + __entry->trans_fn, + __entry->idx, + __entry->ref, + __entry->preserve, + bch2_btree_id_str(__entry->btree_id), + __entry->pos_inode, + __entry->pos_offset, + __entry->pos_snapshot, + __entry->level, + __entry->locks_want, + (__entry->nodes_locked >> 6) & 3, + (__entry->nodes_locked >> 4) & 3, + (__entry->nodes_locked >> 2) & 3, + (__entry->nodes_locked >> 0) & 3, + __entry->node3, + __entry->node2, + __entry->node1, + __entry->node0) +); + +DEFINE_EVENT(btree_path_traverse, btree_path_traverse_start, + TP_PROTO(struct btree_trans *trans, + struct btree_path *path), + TP_ARGS(trans, path) +); + +DEFINE_EVENT(btree_path_traverse, btree_path_traverse_end, + TP_PROTO(struct btree_trans *trans, struct btree_path *path), + TP_ARGS(trans, path) +); + +TRACE_EVENT(btree_path_set_pos, + TP_PROTO(struct btree_trans *trans, + struct btree_path *path, + struct bpos *new_pos), + TP_ARGS(trans, path, new_pos), + + TP_STRUCT__entry( + __field(btree_path_idx_t, idx ) + __field(u8, ref ) + __field(u8, preserve ) + __field(u8, btree_id ) + TRACE_BPOS_entries(old_pos) + TRACE_BPOS_entries(new_pos) + __field(u8, locks_want ) + __field(u8, nodes_locked ) + __array(char, node0, 24 ) + __array(char, node1, 24 ) + __array(char, node2, 24 ) + __array(char, node3, 24 ) + ), + + TP_fast_assign( + __entry->idx = path - trans->paths; + __entry->ref = path->ref; + __entry->preserve = path->preserve; + __entry->btree_id = path->btree_id; + TRACE_BPOS_assign(old_pos, path->pos); + TRACE_BPOS_assign(new_pos, *new_pos); + + __entry->nodes_locked = path->nodes_locked; + struct btree *b = path->l[0].b; + if (IS_ERR(b)) + strscpy(__entry->node0, bch2_err_str(PTR_ERR(b)), sizeof(__entry->node0)); + else + scnprintf(__entry->node0, sizeof(__entry->node0), "%px", &b->c); + b = path->l[1].b; + if (IS_ERR(b)) + strscpy(__entry->node1, bch2_err_str(PTR_ERR(b)), sizeof(__entry->node0)); + else + scnprintf(__entry->node1, sizeof(__entry->node0), "%px", &b->c); + b = path->l[2].b; + if (IS_ERR(b)) + strscpy(__entry->node2, bch2_err_str(PTR_ERR(b)), sizeof(__entry->node0)); + else + scnprintf(__entry->node2, sizeof(__entry->node0), "%px", &b->c); + b = path->l[3].b; + if (IS_ERR(b)) + strscpy(__entry->node3, bch2_err_str(PTR_ERR(b)), sizeof(__entry->node0)); + else + scnprintf(__entry->node3, sizeof(__entry->node0), "%px", &b->c); + ), + + TP_printk("\npath %3u ref %u preserve %u btree %s %llu:%llu:%u -> %llu:%llu:%u\n" + "locks %u %u %u %u node %s %s %s %s", + __entry->idx, + __entry->ref, + __entry->preserve, + bch2_btree_id_str(__entry->btree_id), + __entry->old_pos_inode, + __entry->old_pos_offset, + __entry->old_pos_snapshot, + __entry->new_pos_inode, + __entry->new_pos_offset, + __entry->new_pos_snapshot, + (__entry->nodes_locked >> 6) & 3, + (__entry->nodes_locked >> 4) & 3, + (__entry->nodes_locked >> 2) & 3, + (__entry->nodes_locked >> 0) & 3, + __entry->node3, + __entry->node2, + __entry->node1, + __entry->node0) +); + +TRACE_EVENT(btree_path_free, + TP_PROTO(struct btree_trans *trans, btree_path_idx_t path, struct btree_path *dup), + TP_ARGS(trans, path, dup), + + TP_STRUCT__entry( + __field(btree_path_idx_t, idx ) + __field(u8, preserve ) + __field(u8, should_be_locked) + __field(s8, dup ) + __field(u8, dup_locked ) + ), + + TP_fast_assign( + __entry->idx = path; + __entry->preserve = trans->paths[path].preserve; + __entry->should_be_locked = trans->paths[path].should_be_locked; + __entry->dup = dup ? dup - trans->paths : -1; + __entry->dup_locked = dup ? btree_node_locked(dup, dup->level) : 0; + ), + + TP_printk(" path %3u %c %c dup %2i locked %u", __entry->idx, + __entry->preserve ? 'P' : ' ', + __entry->should_be_locked ? 'S' : ' ', + __entry->dup, + __entry->dup_locked) +); + +TRACE_EVENT(btree_path_free_trans_begin, + TP_PROTO(btree_path_idx_t path), + TP_ARGS(path), + + TP_STRUCT__entry( + __field(btree_path_idx_t, idx ) + ), + + TP_fast_assign( + __entry->idx = path; + ), + + TP_printk(" path %3u", __entry->idx) +); + +#else /* CONFIG_BCACHEFS_PATH_TRACEPOINTS */ +#ifndef _TRACE_BCACHEFS_H + +static inline void trace_update_by_path(struct btree_trans *trans, struct btree_path *path, + struct btree_insert_entry *i, bool overwrite) {} +static inline void trace_btree_path_lock(struct btree_trans *trans, unsigned long caller_ip, struct btree_bkey_cached_common *b) {} +static inline void trace_btree_path_get_ll(struct btree_trans *trans, struct btree_path *path) {} +static inline void trace_btree_path_put_ll(struct btree_trans *trans, struct btree_path *path) {} +static inline void trace_btree_path_should_be_locked(struct btree_trans *trans, struct btree_path *path) {} +static inline void trace_btree_path_alloc(struct btree_trans *trans, struct btree_path *path) {} +static inline void trace_btree_path_get(struct btree_trans *trans, struct btree_path *path, struct bpos *new_pos) {} +static inline void trace_btree_path_clone(struct btree_trans *trans, struct btree_path *path, struct btree_path *new) {} +static inline void trace_btree_path_save_pos(struct btree_trans *trans, struct btree_path *path, struct btree_path *new) {} +static inline void trace_btree_path_traverse_start(struct btree_trans *trans, struct btree_path *path) {} +static inline void trace_btree_path_traverse_end(struct btree_trans *trans, struct btree_path *path) {} +static inline void trace_btree_path_set_pos(struct btree_trans *trans, struct btree_path *path, struct bpos *new_pos) {} +static inline void trace_btree_path_free(struct btree_trans *trans, btree_path_idx_t path, struct btree_path *dup) {} +static inline void trace_btree_path_free_trans_begin(btree_path_idx_t path) {} + +#endif +#endif /* CONFIG_BCACHEFS_PATH_TRACEPOINTS */ + +#define _TRACE_BCACHEFS_H #endif /* _TRACE_BCACHEFS_H */ /* This part must be outside protection */ From 804baca7458a45ab70b985a89ed5349c182b4837 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 17 Jul 2024 11:42:11 -0400 Subject: [PATCH 141/302] bcachefs: kill bch2_btree_iter_peek_and_restart() dead code Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_iter.c | 1 + fs/bcachefs/btree_iter.h | 14 -------------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/fs/bcachefs/btree_iter.c b/fs/bcachefs/btree_iter.c index c71efbdb3f07..bfe9f0c1e1be 100644 --- a/fs/bcachefs/btree_iter.c +++ b/fs/bcachefs/btree_iter.c @@ -2726,6 +2726,7 @@ struct bkey_s_c bch2_btree_iter_prev_slot(struct btree_iter *iter) return bch2_btree_iter_peek_slot(iter); } +/* Obsolete, but still used by rust wrapper in -tools */ struct bkey_s_c bch2_btree_iter_peek_and_restart_outlined(struct btree_iter *iter) { struct bkey_s_c k; diff --git a/fs/bcachefs/btree_iter.h b/fs/bcachefs/btree_iter.h index 172507b4dead..2ad90f40325a 100644 --- a/fs/bcachefs/btree_iter.h +++ b/fs/bcachefs/btree_iter.h @@ -832,20 +832,6 @@ transaction_restart: \ struct bkey_s_c bch2_btree_iter_peek_and_restart_outlined(struct btree_iter *); -static inline struct bkey_s_c -__bch2_btree_iter_peek_and_restart(struct btree_trans *trans, - struct btree_iter *iter, unsigned flags) -{ - struct bkey_s_c k; - - while (btree_trans_too_many_iters(trans) || - (k = bch2_btree_iter_peek_type(iter, flags), - bch2_err_matches(bkey_err(k), BCH_ERR_transaction_restart))) - bch2_trans_begin(trans); - - return k; -} - #define for_each_btree_key_upto_norestart(_trans, _iter, _btree_id, \ _start, _end, _flags, _k, _ret) \ for (bch2_trans_iter_init((_trans), &(_iter), (_btree_id), \ From 7e7595723c659e91d1f2597574b6fe77b67858bf Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 17 Jul 2024 11:47:01 -0400 Subject: [PATCH 142/302] bcachefs: bchfs_read(): call trans_begin() on every loop iter Same as the recent change for __bch2_read(); also, kill now unnecessary btree_trans_too_many_iters() calls. Signed-off-by: Kent Overstreet --- fs/bcachefs/fs-io-buffered.c | 39 +++++++++++++----------------------- fs/bcachefs/io_read.c | 4 ---- 2 files changed, 14 insertions(+), 29 deletions(-) diff --git a/fs/bcachefs/fs-io-buffered.c b/fs/bcachefs/fs-io-buffered.c index 9fe1285908fa..99fef9342199 100644 --- a/fs/bcachefs/fs-io-buffered.c +++ b/fs/bcachefs/fs-io-buffered.c @@ -151,7 +151,6 @@ static void bchfs_read(struct btree_trans *trans, struct bkey_buf sk; int flags = BCH_READ_RETRY_IF_STALE| BCH_READ_MAY_PROMOTE; - u32 snapshot; int ret = 0; rbio->c = c; @@ -159,29 +158,23 @@ static void bchfs_read(struct btree_trans *trans, rbio->subvol = inum.subvol; bch2_bkey_buf_init(&sk); -retry: bch2_trans_begin(trans); - iter = (struct btree_iter) { NULL }; - - ret = bch2_subvolume_get_snapshot(trans, inum.subvol, &snapshot); - if (ret) - goto err; - bch2_trans_iter_init(trans, &iter, BTREE_ID_extents, - SPOS(inum.inum, rbio->bio.bi_iter.bi_sector, snapshot), + POS(inum.inum, rbio->bio.bi_iter.bi_sector), BTREE_ITER_slots); while (1) { struct bkey_s_c k; unsigned bytes, sectors, offset_into_extent; enum btree_id data_btree = BTREE_ID_extents; - /* - * read_extent -> io_time_reset may cause a transaction restart - * without returning an error, we need to check for that here: - */ - ret = bch2_trans_relock(trans); + bch2_trans_begin(trans); + + u32 snapshot; + ret = bch2_subvolume_get_snapshot(trans, inum.subvol, &snapshot); if (ret) - break; + goto err; + + bch2_btree_iter_set_snapshot(&iter, snapshot); bch2_btree_iter_set_pos(&iter, POS(inum.inum, rbio->bio.bi_iter.bi_sector)); @@ -189,7 +182,7 @@ retry: k = bch2_btree_iter_peek_slot(&iter); ret = bkey_err(k); if (ret) - break; + goto err; offset_into_extent = iter.pos.offset - bkey_start_offset(k.k); @@ -200,7 +193,7 @@ retry: ret = bch2_read_indirect_extent(trans, &data_btree, &offset_into_extent, &sk); if (ret) - break; + goto err; k = bkey_i_to_s_c(sk.k); @@ -210,7 +203,7 @@ retry: ret = readpage_bio_extend(trans, readpages_iter, &rbio->bio, sectors, extent_partial_reads_expensive(k)); if (ret) - break; + goto err; } bytes = min(sectors, bio_sectors(&rbio->bio)) << 9; @@ -229,17 +222,13 @@ retry: swap(rbio->bio.bi_iter.bi_size, bytes); bio_advance(&rbio->bio, bytes); - - ret = btree_trans_too_many_iters(trans); - if (ret) +err: + if (ret && + !bch2_err_matches(ret, BCH_ERR_transaction_restart)) break; } -err: bch2_trans_iter_exit(trans, &iter); - if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) - goto retry; - if (ret) { bch_err_inum_offset_ratelimited(c, iter.pos.inode, diff --git a/fs/bcachefs/io_read.c b/fs/bcachefs/io_read.c index 7ee3b75480df..808017873d07 100644 --- a/fs/bcachefs/io_read.c +++ b/fs/bcachefs/io_read.c @@ -1214,10 +1214,6 @@ void __bch2_read(struct bch_fs *c, struct bch_read_bio *rbio, swap(bvec_iter.bi_size, bytes); bio_advance_iter(&rbio->bio, &bvec_iter, bytes); - - ret = btree_trans_too_many_iters(trans); - if (ret) - goto err; err: if (ret && !bch2_err_matches(ret, BCH_ERR_transaction_restart) && From 1a3158ece59c7444b98124805d142be13c9d560b Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 17 Jul 2024 11:50:54 -0400 Subject: [PATCH 143/302] bcachefs: bch2_fiemap(): call trans_begin() on every loop iter Signed-off-by: Kent Overstreet --- fs/bcachefs/fs.c | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/fs/bcachefs/fs.c b/fs/bcachefs/fs.c index d90b3082a86f..2de776d31b52 100644 --- a/fs/bcachefs/fs.c +++ b/fs/bcachefs/fs.c @@ -1095,7 +1095,6 @@ static int bch2_fiemap(struct inode *vinode, struct fiemap_extent_info *info, struct bkey_buf cur, prev; unsigned offset_into_extent, sectors; bool have_extent = false; - u32 snapshot; int ret = 0; ret = fiemap_prep(&ei->v, info, start, &len, FIEMAP_FLAG_SYNC); @@ -1111,21 +1110,30 @@ static int bch2_fiemap(struct inode *vinode, struct fiemap_extent_info *info, bch2_bkey_buf_init(&cur); bch2_bkey_buf_init(&prev); trans = bch2_trans_get(c); -retry: - bch2_trans_begin(trans); - - ret = bch2_subvolume_get_snapshot(trans, ei->ei_inum.subvol, &snapshot); - if (ret) - goto err; bch2_trans_iter_init(trans, &iter, BTREE_ID_extents, - SPOS(ei->v.i_ino, start, snapshot), 0); + POS(ei->v.i_ino, start), 0); - while (!(ret = btree_trans_too_many_iters(trans)) && - (k = bch2_btree_iter_peek_upto(&iter, end)).k && - !(ret = bkey_err(k))) { + while (true) { enum btree_id data_btree = BTREE_ID_extents; + bch2_trans_begin(trans); + + u32 snapshot; + ret = bch2_subvolume_get_snapshot(trans, ei->ei_inum.subvol, &snapshot); + if (ret) + goto err; + + bch2_btree_iter_set_snapshot(&iter, snapshot); + + k = bch2_btree_iter_peek_upto(&iter, end); + ret = bkey_err(k); + if (ret) + goto err; + + if (!k.k) + break; + if (!bkey_extent_is_data(k.k) && k.k->type != KEY_TYPE_reservation) { bch2_btree_iter_advance(&iter); @@ -1169,16 +1177,12 @@ retry: bch2_btree_iter_set_pos(&iter, POS(iter.pos.inode, iter.pos.offset + sectors)); - - ret = bch2_trans_relock(trans); - if (ret) +err: + if (ret && + !bch2_err_matches(ret, BCH_ERR_transaction_restart)) break; } - start = iter.pos.offset; bch2_trans_iter_exit(trans, &iter); -err: - if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) - goto retry; if (!ret && have_extent) { bch2_trans_unlock(trans); From 0215b91804621d443b96cee83e7e7d18445e856b Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 17 Jul 2024 12:59:51 -0400 Subject: [PATCH 144/302] bcachefs: for_each_btree_key_in_subvolume_upto() New helper for looping over keys in a given subvolume Signed-off-by: Kent Overstreet --- fs/bcachefs/subvolume.h | 45 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/fs/bcachefs/subvolume.h b/fs/bcachefs/subvolume.h index a8299ba2cab2..e62f876541fe 100644 --- a/fs/bcachefs/subvolume.h +++ b/fs/bcachefs/subvolume.h @@ -31,6 +31,51 @@ int bch2_subvolume_get_snapshot(struct btree_trans *, u32, u32 *); int bch2_subvol_is_ro_trans(struct btree_trans *, u32); int bch2_subvol_is_ro(struct bch_fs *, u32); +static inline struct bkey_s_c +bch2_btree_iter_peek_in_subvolume_upto_type(struct btree_iter *iter, struct bpos end, + u32 subvolid, unsigned flags) +{ + u32 snapshot; + int ret = bch2_subvolume_get_snapshot(iter->trans, subvolid, &snapshot); + if (ret) + return bkey_s_c_err(ret); + + bch2_btree_iter_set_snapshot(iter, snapshot); + return bch2_btree_iter_peek_upto_type(iter, end, flags); +} + +#define for_each_btree_key_in_subvolume_upto_continue(_trans, _iter, \ + _end, _subvolid, _flags, _k, _do) \ +({ \ + struct bkey_s_c _k; \ + int _ret3 = 0; \ + \ + do { \ + _ret3 = lockrestart_do(_trans, ({ \ + (_k) = bch2_btree_iter_peek_in_subvolume_upto_type(&(_iter), \ + _end, _subvolid, (_flags)); \ + if (!(_k).k) \ + break; \ + \ + bkey_err(_k) ?: (_do); \ + })); \ + } while (!_ret3 && bch2_btree_iter_advance(&(_iter))); \ + \ + bch2_trans_iter_exit((_trans), &(_iter)); \ + _ret3; \ +}) + +#define for_each_btree_key_in_subvolume_upto(_trans, _iter, _btree_id, \ + _start, _end, _subvolid, _flags, _k, _do) \ +({ \ + struct btree_iter _iter; \ + bch2_trans_iter_init((_trans), &(_iter), (_btree_id), \ + (_start), (_flags)); \ + \ + for_each_btree_key_in_subvolume_upto_continue(_trans, _iter, \ + _end, _subvolid, _flags, _k, _do); \ +}) + int bch2_delete_dead_snapshots(struct bch_fs *); void bch2_delete_dead_snapshots_async(struct bch_fs *); From efdb77a25baf78318bbf893e754a07158ab1bd01 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 17 Jul 2024 13:24:28 -0400 Subject: [PATCH 145/302] bcachefs: bch2_readdir() -> for_each_btree_key_in_subvolume_upto Signed-off-by: Kent Overstreet --- fs/bcachefs/dirent.c | 66 ++++++++++++-------------------------------- 1 file changed, 17 insertions(+), 49 deletions(-) diff --git a/fs/bcachefs/dirent.c b/fs/bcachefs/dirent.c index 32bfdf19289a..84dd4a879d98 100644 --- a/fs/bcachefs/dirent.c +++ b/fs/bcachefs/dirent.c @@ -552,62 +552,30 @@ static int bch2_dir_emit(struct dir_context *ctx, struct bkey_s_c_dirent d, subv int bch2_readdir(struct bch_fs *c, subvol_inum inum, struct dir_context *ctx) { - struct btree_trans *trans = bch2_trans_get(c); - struct btree_iter iter; - struct bkey_s_c k; - subvol_inum target; - u32 snapshot; struct bkey_buf sk; - int ret; - bch2_bkey_buf_init(&sk); -retry: - bch2_trans_begin(trans); - ret = bch2_subvolume_get_snapshot(trans, inum.subvol, &snapshot); - if (ret) - goto err; + int ret = bch2_trans_run(c, + for_each_btree_key_in_subvolume_upto(trans, iter, BTREE_ID_dirents, + POS(inum.inum, ctx->pos), + POS(inum.inum, U64_MAX), + inum.subvol, 0, k, ({ + if (k.k->type != KEY_TYPE_dirent) + continue; - for_each_btree_key_upto_norestart(trans, iter, BTREE_ID_dirents, - SPOS(inum.inum, ctx->pos, snapshot), - POS(inum.inum, U64_MAX), 0, k, ret) { - if (k.k->type != KEY_TYPE_dirent) - continue; + /* dir_emit() can fault and block: */ + bch2_bkey_buf_reassemble(&sk, c, k); + struct bkey_s_c_dirent dirent = bkey_i_to_s_c_dirent(sk.k); - /* dir_emit() can fault and block: */ - bch2_bkey_buf_reassemble(&sk, c, k); - struct bkey_s_c_dirent dirent = bkey_i_to_s_c_dirent(sk.k); + subvol_inum target; + int ret2 = bch2_dirent_read_target(trans, inum, dirent, &target); + if (ret2 > 0) + continue; - ret = bch2_dirent_read_target(trans, inum, dirent, &target); - if (ret < 0) - break; - if (ret) - continue; + ret2 ?: drop_locks_do(trans, bch2_dir_emit(ctx, dirent, target)); + }))); - /* - * read_target looks up subvolumes, we can overflow paths if the - * directory has many subvolumes in it - * - * XXX: btree_trans_too_many_iters() is something we'd like to - * get rid of, and there's no good reason to be using it here - * except that we don't yet have a for_each_btree_key() helper - * that does subvolume_get_snapshot(). - */ - ret = drop_locks_do(trans, - bch2_dir_emit(ctx, dirent, target)) ?: - btree_trans_too_many_iters(trans); - if (ret) { - ret = ret < 0 ? ret : 0; - break; - } - } - bch2_trans_iter_exit(trans, &iter); -err: - if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) - goto retry; - - bch2_trans_put(trans); bch2_bkey_buf_exit(&sk, c); - return ret; + return ret < 0 ? ret : 0; } From 3da106cd1bde9654bcc7837aed62850519684526 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 17 Jul 2024 13:24:28 -0400 Subject: [PATCH 146/302] bcachefs: bch2_xattr_list() -> for_each_btree_key_in_subvolume_upto Signed-off-by: Kent Overstreet --- fs/bcachefs/xattr.c | 55 ++++++++++----------------------------------- 1 file changed, 12 insertions(+), 43 deletions(-) diff --git a/fs/bcachefs/xattr.c b/fs/bcachefs/xattr.c index 4b5898d3eda0..9f810fb36ba1 100644 --- a/fs/bcachefs/xattr.c +++ b/fs/bcachefs/xattr.c @@ -295,54 +295,23 @@ ssize_t bch2_xattr_list(struct dentry *dentry, char *buffer, size_t buffer_size) { struct bch_fs *c = dentry->d_sb->s_fs_info; struct bch_inode_info *inode = to_bch_ei(dentry->d_inode); - struct btree_trans *trans = bch2_trans_get(c); - struct btree_iter iter; - struct bkey_s_c k; struct xattr_buf buf = { .buf = buffer, .len = buffer_size }; u64 offset = 0, inum = inode->ei_inode.bi_inum; - u32 snapshot; - int ret; -retry: - bch2_trans_begin(trans); - iter = (struct btree_iter) { NULL }; - ret = bch2_subvolume_get_snapshot(trans, inode->ei_inum.subvol, &snapshot); - if (ret) - goto err; + int ret = bch2_trans_run(c, + for_each_btree_key_in_subvolume_upto(trans, iter, BTREE_ID_xattrs, + POS(inum, offset), + POS(inum, U64_MAX), + inode->ei_inum.subvol, 0, k, ({ + if (k.k->type != KEY_TYPE_xattr) + continue; - for_each_btree_key_upto_norestart(trans, iter, BTREE_ID_xattrs, - SPOS(inum, offset, snapshot), - POS(inum, U64_MAX), 0, k, ret) { - if (k.k->type != KEY_TYPE_xattr) - continue; + bch2_xattr_emit(dentry, bkey_s_c_to_xattr(k).v, &buf); + }))) ?: + bch2_xattr_list_bcachefs(c, &inode->ei_inode, &buf, false) ?: + bch2_xattr_list_bcachefs(c, &inode->ei_inode, &buf, true); - ret = bch2_xattr_emit(dentry, bkey_s_c_to_xattr(k).v, &buf); - if (ret) - break; - } - - offset = iter.pos.offset; - bch2_trans_iter_exit(trans, &iter); -err: - if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) - goto retry; - - bch2_trans_put(trans); - - if (ret) - goto out; - - ret = bch2_xattr_list_bcachefs(c, &inode->ei_inode, &buf, false); - if (ret) - goto out; - - ret = bch2_xattr_list_bcachefs(c, &inode->ei_inode, &buf, true); - if (ret) - goto out; - - return buf.used; -out: - return bch2_err_class(ret); + return ret ? bch2_err_class(ret) : buf.used; } static int bch2_xattr_get_handler(const struct xattr_handler *handler, From 9f9e7f50af0d62b357aff1d0c4afc8fe96bc8953 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 17 Jul 2024 13:26:54 -0400 Subject: [PATCH 147/302] bcachefs: bch2_seek_data() -> for_each_btree_key_in_subvolume_upto Signed-off-by: Kent Overstreet --- fs/bcachefs/fs-io.c | 41 ++++++++++++----------------------------- 1 file changed, 12 insertions(+), 29 deletions(-) diff --git a/fs/bcachefs/fs-io.c b/fs/bcachefs/fs-io.c index a5018cbdb376..2c49051308a2 100644 --- a/fs/bcachefs/fs-io.c +++ b/fs/bcachefs/fs-io.c @@ -942,42 +942,25 @@ static loff_t bch2_seek_data(struct file *file, u64 offset) { struct bch_inode_info *inode = file_bch_inode(file); struct bch_fs *c = inode->v.i_sb->s_fs_info; - struct btree_trans *trans; - struct btree_iter iter; - struct bkey_s_c k; subvol_inum inum = inode_inum(inode); u64 isize, next_data = MAX_LFS_FILESIZE; - u32 snapshot; - int ret; isize = i_size_read(&inode->v); if (offset >= isize) return -ENXIO; - trans = bch2_trans_get(c); -retry: - bch2_trans_begin(trans); - - ret = bch2_subvolume_get_snapshot(trans, inum.subvol, &snapshot); - if (ret) - goto err; - - for_each_btree_key_upto_norestart(trans, iter, BTREE_ID_extents, - SPOS(inode->v.i_ino, offset >> 9, snapshot), - POS(inode->v.i_ino, U64_MAX), - 0, k, ret) { - if (bkey_extent_is_data(k.k)) { - next_data = max(offset, bkey_start_offset(k.k) << 9); - break; - } else if (k.k->p.offset >> 9 > isize) - break; - } - bch2_trans_iter_exit(trans, &iter); -err: - if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) - goto retry; - - bch2_trans_put(trans); + int ret = bch2_trans_run(c, + for_each_btree_key_in_subvolume_upto(trans, iter, BTREE_ID_extents, + POS(inode->v.i_ino, offset >> 9), + POS(inode->v.i_ino, U64_MAX), + inum.subvol, 0, k, ({ + if (bkey_extent_is_data(k.k)) { + next_data = max(offset, bkey_start_offset(k.k) << 9); + break; + } else if (k.k->p.offset >> 9 > isize) + break; + 0; + }))); if (ret) return ret; From 330405057fae56f2f21055851a5a816626679226 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 17 Jul 2024 13:28:23 -0400 Subject: [PATCH 148/302] bcachefs: bch2_seek_hole() -> for_each_btree_key_in_subvolume_upto Signed-off-by: Kent Overstreet --- fs/bcachefs/fs-io.c | 54 ++++++++++++++++----------------------------- 1 file changed, 19 insertions(+), 35 deletions(-) diff --git a/fs/bcachefs/fs-io.c b/fs/bcachefs/fs-io.c index 2c49051308a2..b0ecc756d766 100644 --- a/fs/bcachefs/fs-io.c +++ b/fs/bcachefs/fs-io.c @@ -978,50 +978,34 @@ static loff_t bch2_seek_hole(struct file *file, u64 offset) { struct bch_inode_info *inode = file_bch_inode(file); struct bch_fs *c = inode->v.i_sb->s_fs_info; - struct btree_trans *trans; - struct btree_iter iter; - struct bkey_s_c k; subvol_inum inum = inode_inum(inode); u64 isize, next_hole = MAX_LFS_FILESIZE; - u32 snapshot; - int ret; isize = i_size_read(&inode->v); if (offset >= isize) return -ENXIO; - trans = bch2_trans_get(c); -retry: - bch2_trans_begin(trans); - - ret = bch2_subvolume_get_snapshot(trans, inum.subvol, &snapshot); - if (ret) - goto err; - - for_each_btree_key_norestart(trans, iter, BTREE_ID_extents, - SPOS(inode->v.i_ino, offset >> 9, snapshot), - BTREE_ITER_slots, k, ret) { - if (k.k->p.inode != inode->v.i_ino) { - next_hole = bch2_seek_pagecache_hole(&inode->v, - offset, MAX_LFS_FILESIZE, 0, false); - break; - } else if (!bkey_extent_is_data(k.k)) { - next_hole = bch2_seek_pagecache_hole(&inode->v, - max(offset, bkey_start_offset(k.k) << 9), - k.k->p.offset << 9, 0, false); - - if (next_hole < k.k->p.offset << 9) + int ret = bch2_trans_run(c, + for_each_btree_key_in_subvolume_upto(trans, iter, BTREE_ID_extents, + POS(inode->v.i_ino, offset >> 9), + POS(inode->v.i_ino, U64_MAX), + inum.subvol, BTREE_ITER_slots, k, ({ + if (k.k->p.inode != inode->v.i_ino) { + next_hole = bch2_seek_pagecache_hole(&inode->v, + offset, MAX_LFS_FILESIZE, 0, false); break; - } else { - offset = max(offset, bkey_start_offset(k.k) << 9); - } - } - bch2_trans_iter_exit(trans, &iter); -err: - if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) - goto retry; + } else if (!bkey_extent_is_data(k.k)) { + next_hole = bch2_seek_pagecache_hole(&inode->v, + max(offset, bkey_start_offset(k.k) << 9), + k.k->p.offset << 9, 0, false); - bch2_trans_put(trans); + if (next_hole < k.k->p.offset << 9) + break; + } else { + offset = max(offset, bkey_start_offset(k.k) << 9); + } + 0; + }))); if (ret) return ret; From c95285d17ec6072054bb4e0423d2d0eab67f6aa6 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 17 Jul 2024 13:30:23 -0400 Subject: [PATCH 149/302] bcachefs: range_has_data() -> for_each_btree_key_in_subvolume_upto Signed-off-by: Kent Overstreet --- fs/bcachefs/fs-io.c | 29 +++++------------------------ 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/fs/bcachefs/fs-io.c b/fs/bcachefs/fs-io.c index b0ecc756d766..4359fde86603 100644 --- a/fs/bcachefs/fs-io.c +++ b/fs/bcachefs/fs-io.c @@ -221,30 +221,11 @@ static inline int range_has_data(struct bch_fs *c, u32 subvol, struct bpos start, struct bpos end) { - struct btree_trans *trans = bch2_trans_get(c); - struct btree_iter iter; - struct bkey_s_c k; - int ret = 0; -retry: - bch2_trans_begin(trans); - - ret = bch2_subvolume_get_snapshot(trans, subvol, &start.snapshot); - if (ret) - goto err; - - for_each_btree_key_upto_norestart(trans, iter, BTREE_ID_extents, start, end, 0, k, ret) - if (bkey_extent_is_data(k.k) && !bkey_extent_is_unwritten(k)) { - ret = 1; - break; - } - start = iter.pos; - bch2_trans_iter_exit(trans, &iter); -err: - if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) - goto retry; - - bch2_trans_put(trans); - return ret; + return bch2_trans_run(c, + for_each_btree_key_in_subvolume_upto(trans, iter, BTREE_ID_extents, start, end, + subvol, 0, k, ({ + bkey_extent_is_data(k.k) && !bkey_extent_is_unwritten(k); + }))); } static int __bch2_truncate_folio(struct bch_inode_info *inode, From 093dd55d192c01f055476d7c6f017ca84fc9e167 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 17 Jul 2024 13:34:35 -0400 Subject: [PATCH 150/302] bcachefs: bch2_folio_set() -> for_each_btree_key_in_subvolume_upto Signed-off-by: Kent Overstreet --- fs/bcachefs/fs-io-pagecache.c | 78 +++++++++++++---------------------- 1 file changed, 29 insertions(+), 49 deletions(-) diff --git a/fs/bcachefs/fs-io-pagecache.c b/fs/bcachefs/fs-io-pagecache.c index a9cc5cad9cc9..af3a24546aa3 100644 --- a/fs/bcachefs/fs-io-pagecache.c +++ b/fs/bcachefs/fs-io-pagecache.c @@ -182,18 +182,11 @@ static void __bch2_folio_set(struct folio *folio, int bch2_folio_set(struct bch_fs *c, subvol_inum inum, struct folio **fs, unsigned nr_folios) { - struct btree_trans *trans; - struct btree_iter iter; - struct bkey_s_c k; - struct bch_folio *s; u64 offset = folio_sector(fs[0]); - unsigned folio_idx; - u32 snapshot; bool need_set = false; - int ret; - for (folio_idx = 0; folio_idx < nr_folios; folio_idx++) { - s = bch2_folio_create(fs[folio_idx], GFP_KERNEL); + for (unsigned folio_idx = 0; folio_idx < nr_folios; folio_idx++) { + struct bch_folio *s = bch2_folio_create(fs[folio_idx], GFP_KERNEL); if (!s) return -ENOMEM; @@ -203,53 +196,40 @@ int bch2_folio_set(struct bch_fs *c, subvol_inum inum, if (!need_set) return 0; - folio_idx = 0; - trans = bch2_trans_get(c); -retry: - bch2_trans_begin(trans); + unsigned folio_idx = 0; - ret = bch2_subvolume_get_snapshot(trans, inum.subvol, &snapshot); - if (ret) - goto err; + return bch2_trans_run(c, + for_each_btree_key_in_subvolume_upto(trans, iter, BTREE_ID_extents, + POS(inum.inum, offset), + POS(inum.inum, U64_MAX), + inum.subvol, BTREE_ITER_slots, k, ({ + unsigned nr_ptrs = bch2_bkey_nr_ptrs_fully_allocated(k); + unsigned state = bkey_to_sector_state(k); - for_each_btree_key_norestart(trans, iter, BTREE_ID_extents, - SPOS(inum.inum, offset, snapshot), - BTREE_ITER_slots, k, ret) { - unsigned nr_ptrs = bch2_bkey_nr_ptrs_fully_allocated(k); - unsigned state = bkey_to_sector_state(k); + while (folio_idx < nr_folios) { + struct folio *folio = fs[folio_idx]; + u64 folio_start = folio_sector(folio); + u64 folio_end = folio_end_sector(folio); + unsigned folio_offset = max(bkey_start_offset(k.k), folio_start) - + folio_start; + unsigned folio_len = min(k.k->p.offset, folio_end) - + folio_offset - folio_start; - while (folio_idx < nr_folios) { - struct folio *folio = fs[folio_idx]; - u64 folio_start = folio_sector(folio); - u64 folio_end = folio_end_sector(folio); - unsigned folio_offset = max(bkey_start_offset(k.k), folio_start) - - folio_start; - unsigned folio_len = min(k.k->p.offset, folio_end) - - folio_offset - folio_start; + BUG_ON(k.k->p.offset < folio_start); + BUG_ON(bkey_start_offset(k.k) > folio_end); - BUG_ON(k.k->p.offset < folio_start); - BUG_ON(bkey_start_offset(k.k) > folio_end); + if (!bch2_folio(folio)->uptodate) + __bch2_folio_set(folio, folio_offset, folio_len, nr_ptrs, state); - if (!bch2_folio(folio)->uptodate) - __bch2_folio_set(folio, folio_offset, folio_len, nr_ptrs, state); + if (k.k->p.offset < folio_end) + break; + folio_idx++; + } - if (k.k->p.offset < folio_end) + if (folio_idx == nr_folios) break; - folio_idx++; - } - - if (folio_idx == nr_folios) - break; - } - - offset = iter.pos.offset; - bch2_trans_iter_exit(trans, &iter); -err: - if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) - goto retry; - bch2_trans_put(trans); - - return ret; + 0; + }))); } void bch2_bio_page_state_set(struct bio *bio, struct bkey_s_c k) From 668c95515580be9f10b58dcd64cc28c03a733da8 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 17 Jul 2024 13:24:28 -0400 Subject: [PATCH 151/302] bcachefs: quota_reserve_range() -> for_each_btree_key_in_subvolume_upto Signed-off-by: Kent Overstreet --- fs/bcachefs/fs-io.c | 46 ++++++++++++++------------------------------- 1 file changed, 14 insertions(+), 32 deletions(-) diff --git a/fs/bcachefs/fs-io.c b/fs/bcachefs/fs-io.c index 4359fde86603..71d0fa387509 100644 --- a/fs/bcachefs/fs-io.c +++ b/fs/bcachefs/fs-io.c @@ -794,41 +794,23 @@ static int quota_reserve_range(struct bch_inode_info *inode, u64 start, u64 end) { struct bch_fs *c = inode->v.i_sb->s_fs_info; - struct btree_trans *trans = bch2_trans_get(c); - struct btree_iter iter; - struct bkey_s_c k; - u32 snapshot; u64 sectors = end - start; - u64 pos = start; - int ret; -retry: - bch2_trans_begin(trans); - ret = bch2_subvolume_get_snapshot(trans, inode->ei_inum.subvol, &snapshot); - if (ret) - goto err; + int ret = bch2_trans_run(c, + for_each_btree_key_in_subvolume_upto(trans, iter, + BTREE_ID_extents, + POS(inode->v.i_ino, start), + POS(inode->v.i_ino, end - 1), + inode->ei_inum.subvol, 0, k, ({ + if (bkey_extent_is_allocation(k.k)) { + u64 s = min(end, k.k->p.offset) - + max(start, bkey_start_offset(k.k)); + BUG_ON(s > sectors); + sectors -= s; + } - bch2_trans_iter_init(trans, &iter, BTREE_ID_extents, - SPOS(inode->v.i_ino, pos, snapshot), 0); - - while (!(ret = btree_trans_too_many_iters(trans)) && - (k = bch2_btree_iter_peek_upto(&iter, POS(inode->v.i_ino, end - 1))).k && - !(ret = bkey_err(k))) { - if (bkey_extent_is_allocation(k.k)) { - u64 s = min(end, k.k->p.offset) - - max(start, bkey_start_offset(k.k)); - BUG_ON(s > sectors); - sectors -= s; - } - bch2_btree_iter_advance(&iter); - } - pos = iter.pos.offset; - bch2_trans_iter_exit(trans, &iter); -err: - if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) - goto retry; - - bch2_trans_put(trans); + 0; + }))); return ret ?: bch2_quota_reservation_add(c, inode, res, sectors, true); } From 23fcd5f40aeff17e7a97d5fd24e0303799f6ece7 Mon Sep 17 00:00:00 2001 From: Julian Sun Date: Sun, 21 Jul 2024 08:43:24 -0400 Subject: [PATCH 152/302] bcachefs: remove the unused macro definition macro bch2_kthread_wait_event_ioclock_timeout is no longer used, let's remove it. The patch has passed compilation test. Signed-off-by: Julian Sun Signed-off-by: Kent Overstreet --- fs/bcachefs/clock.h | 9 --------- 1 file changed, 9 deletions(-) diff --git a/fs/bcachefs/clock.h b/fs/bcachefs/clock.h index 85c975dfbcfe..82c79c8baf92 100644 --- a/fs/bcachefs/clock.h +++ b/fs/bcachefs/clock.h @@ -20,15 +20,6 @@ static inline void bch2_increment_clock(struct bch_fs *c, u64 sectors, void bch2_io_clock_schedule_timeout(struct io_clock *, u64); -#define bch2_kthread_wait_event_ioclock_timeout(condition, clock, timeout)\ -({ \ - long __ret = timeout; \ - might_sleep(); \ - if (!___wait_cond_timeout(condition)) \ - __ret = __wait_event_timeout(wq, condition, timeout); \ - __ret; \ -}) - void bch2_io_timers_to_text(struct printbuf *, struct io_clock *); void bch2_io_clock_exit(struct io_clock *); From ba8c52e2b115b88a5d5836485c1945f86df3ba5d Mon Sep 17 00:00:00 2001 From: Julian Sun Date: Sun, 21 Jul 2024 08:44:24 -0400 Subject: [PATCH 153/302] bcachefs: fix macro definition allocate_dropping_locks_errcode The macro allocate_dropping_locks_errocode accepts a parameter _trans, but it was not used, rather the variable trans was directly used, which may be a local variable inside a function that calls the macros. Signed-off-by: Julian Sun Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_iter.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/bcachefs/btree_iter.h b/fs/bcachefs/btree_iter.h index 2ad90f40325a..140c7b831233 100644 --- a/fs/bcachefs/btree_iter.h +++ b/fs/bcachefs/btree_iter.h @@ -872,7 +872,7 @@ struct bkey_s_c bch2_btree_iter_peek_and_restart_outlined(struct btree_iter *); \ if (bch2_err_matches(_ret, ENOMEM)) { \ _gfp = GFP_KERNEL; \ - _ret = drop_locks_do(trans, _do); \ + _ret = drop_locks_do(_trans, _do); \ } \ _ret; \ }) From 4d05a083b34f8f31a095402e75cc156fd9c3b257 Mon Sep 17 00:00:00 2001 From: Julian Sun Date: Sun, 21 Jul 2024 08:45:47 -0400 Subject: [PATCH 154/302] bcachefs: fix macro definition allocate_dropping_locks The macro allocate_dropping_locks accepts a parameter _trans, but it was not used, rather the variable trans was directly used, which may be a local variable inside a function that calls the macros. Signed-off-by: Julian Sun Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_iter.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/bcachefs/btree_iter.h b/fs/bcachefs/btree_iter.h index 140c7b831233..6d87e57745da 100644 --- a/fs/bcachefs/btree_iter.h +++ b/fs/bcachefs/btree_iter.h @@ -885,7 +885,7 @@ struct bkey_s_c bch2_btree_iter_peek_and_restart_outlined(struct btree_iter *); _ret = 0; \ if (unlikely(!_p)) { \ _gfp = GFP_KERNEL; \ - _ret = drop_locks_do(trans, ((_p = _do), 0)); \ + _ret = drop_locks_do(_trans, ((_p = _do), 0)); \ } \ _p; \ }) From 26c0900d859c47d4bccc05acbc17ac33fa774852 Mon Sep 17 00:00:00 2001 From: Julian Sun Date: Sun, 21 Jul 2024 08:55:20 -0400 Subject: [PATCH 155/302] bcachefs: remove the unused parameter in macro bkey_crc_next In the macro definition of bkey_crc_next, five parameters were accepted, but only four of them were used. Let's remove the unused one. The patch has only passed compilation tests, but it should be fine. Signed-off-by: Julian Sun Signed-off-by: Kent Overstreet --- fs/bcachefs/extents.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/bcachefs/extents.h b/fs/bcachefs/extents.h index 42a7c6d820a0..412e1c5b05a0 100644 --- a/fs/bcachefs/extents.h +++ b/fs/bcachefs/extents.h @@ -357,7 +357,7 @@ out: \ __bkey_for_each_ptr_decode(_k, (_p).start, (_p).end, \ _ptr, _entry) -#define bkey_crc_next(_k, _start, _end, _crc, _iter) \ +#define bkey_crc_next(_k, _end, _crc, _iter) \ ({ \ __bkey_extent_entry_for_each_from(_iter, _end, _iter) \ if (extent_entry_is_crc(_iter)) { \ @@ -372,7 +372,7 @@ out: \ #define __bkey_for_each_crc(_k, _start, _end, _crc, _iter) \ for ((_crc) = bch2_extent_crc_unpack(_k, NULL), \ (_iter) = (_start); \ - bkey_crc_next(_k, _start, _end, _crc, _iter); \ + bkey_crc_next(_k, _end, _crc, _iter); \ (_iter) = extent_entry_next(_iter)) #define bkey_for_each_crc(_k, _p, _crc, _iter) \ From cfd273f1ae8e5bb29a8ff3a7f7b57a55d065e117 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 31 Jul 2024 23:39:49 -0400 Subject: [PATCH 156/302] bcachefs: Move rebalance_status out of sysfs/internal Signed-off-by: Kent Overstreet --- fs/bcachefs/sysfs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/bcachefs/sysfs.c b/fs/bcachefs/sysfs.c index 89da5323cf59..800a3970a242 100644 --- a/fs/bcachefs/sysfs.c +++ b/fs/bcachefs/sysfs.c @@ -515,6 +515,7 @@ struct attribute *bch2_fs_files[] = { &sysfs_btree_write_stats, &sysfs_promote_whole_extents, + &sysfs_rebalance_status, &sysfs_compression_stats, @@ -614,7 +615,6 @@ struct attribute *bch2_fs_internal_files[] = { &sysfs_copy_gc_wait, &sysfs_rebalance_enabled, - &sysfs_rebalance_status, sysfs_pd_controller_files(rebalance), &sysfs_moving_ctxts, From c7652f253a6d590f5263b0f21ff78f58875ab412 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 31 Jul 2024 23:56:04 -0400 Subject: [PATCH 157/302] bcachefs: promote_whole_extents is now a normal option Signed-off-by: Kent Overstreet --- fs/bcachefs/bcachefs.h | 1 - fs/bcachefs/bcachefs_format.h | 2 ++ fs/bcachefs/io_read.c | 2 +- fs/bcachefs/opts.h | 5 +++++ fs/bcachefs/super-io.c | 3 +++ fs/bcachefs/super.c | 1 - fs/bcachefs/sysfs.c | 6 ------ 7 files changed, 11 insertions(+), 9 deletions(-) diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h index ffc90615eba4..a4773b034f91 100644 --- a/fs/bcachefs/bcachefs.h +++ b/fs/bcachefs/bcachefs.h @@ -1086,7 +1086,6 @@ struct bch_fs { u64 __percpu *counters; unsigned copy_gc_enabled:1; - bool promote_whole_extents; struct bch2_time_stats times[BCH_TIME_STAT_NR]; diff --git a/fs/bcachefs/bcachefs_format.h b/fs/bcachefs/bcachefs_format.h index 14ce726bf5a3..8c4addddd07e 100644 --- a/fs/bcachefs/bcachefs_format.h +++ b/fs/bcachefs/bcachefs_format.h @@ -795,6 +795,8 @@ LE64_BITMASK(BCH_SB_HAS_ERRORS, struct bch_sb, flags[0], 60, 61); LE64_BITMASK(BCH_SB_HAS_TOPOLOGY_ERRORS,struct bch_sb, flags[0], 61, 62); LE64_BITMASK(BCH_SB_BIG_ENDIAN, struct bch_sb, flags[0], 62, 63); +LE64_BITMASK(BCH_SB_PROMOTE_WHOLE_EXTENTS, + struct bch_sb, flags[0], 63, 64); LE64_BITMASK(BCH_SB_STR_HASH_TYPE, struct bch_sb, flags[1], 0, 4); LE64_BITMASK(BCH_SB_COMPRESSION_TYPE_LO,struct bch_sb, flags[1], 4, 8); diff --git a/fs/bcachefs/io_read.c b/fs/bcachefs/io_read.c index 808017873d07..ce27ba1f0b54 100644 --- a/fs/bcachefs/io_read.c +++ b/fs/bcachefs/io_read.c @@ -286,7 +286,7 @@ static struct promote_op *promote_alloc(struct btree_trans *trans, */ bool promote_full = (failed || *read_full || - READ_ONCE(c->promote_whole_extents)); + READ_ONCE(c->opts.promote_whole_extents)); /* data might have to be decompressed in the write path: */ unsigned sectors = promote_full ? max(pick->crc.compressed_size, pick->crc.live_size) diff --git a/fs/bcachefs/opts.h b/fs/bcachefs/opts.h index 0138717e9f77..3d83bcdca69d 100644 --- a/fs/bcachefs/opts.h +++ b/fs/bcachefs/opts.h @@ -265,6 +265,11 @@ enum fsck_err_opts { OPT_BOOL(), \ BCH2_NO_SB_OPT, true, \ NULL, "Enable inline data extents") \ + x(promote_whole_extents, u8, \ + OPT_FS|OPT_MOUNT|OPT_RUNTIME, \ + OPT_BOOL(), \ + BCH_SB_PROMOTE_WHOLE_EXTENTS, true, \ + NULL, "Promote whole extents, instead of just part being read")\ x(acl, u8, \ OPT_FS|OPT_FORMAT|OPT_MOUNT, \ OPT_BOOL(), \ diff --git a/fs/bcachefs/super-io.c b/fs/bcachefs/super-io.c index c8c2ccbdfbb5..2a72782d6d17 100644 --- a/fs/bcachefs/super-io.c +++ b/fs/bcachefs/super-io.c @@ -418,6 +418,9 @@ static int bch2_sb_validate(struct bch_sb_handle *disk_sb, if (le16_to_cpu(sb->version) <= bcachefs_metadata_version_disk_accounting_v2 && !BCH_SB_ALLOCATOR_STUCK_TIMEOUT(sb)) SET_BCH_SB_ALLOCATOR_STUCK_TIMEOUT(sb, 30); + + if (le16_to_cpu(sb->version) <= bcachefs_metadata_version_disk_accounting_v2) + SET_BCH_SB_PROMOTE_WHOLE_EXTENTS(sb, true); } for (opt_id = 0; opt_id < bch2_opts_nr; opt_id++) { diff --git a/fs/bcachefs/super.c b/fs/bcachefs/super.c index 6d0ce3d73450..d8adf465f8b8 100644 --- a/fs/bcachefs/super.c +++ b/fs/bcachefs/super.c @@ -811,7 +811,6 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts) c->copy_gc_enabled = 1; c->rebalance.enabled = 1; - c->promote_whole_extents = true; c->journal.flush_write_time = &c->times[BCH_TIME_journal_flush_write]; c->journal.noflush_write_time = &c->times[BCH_TIME_journal_noflush_write]; diff --git a/fs/bcachefs/sysfs.c b/fs/bcachefs/sysfs.c index 800a3970a242..7ff9afcb37b2 100644 --- a/fs/bcachefs/sysfs.c +++ b/fs/bcachefs/sysfs.c @@ -219,7 +219,6 @@ read_attribute(copy_gc_wait); rw_attribute(rebalance_enabled); sysfs_pd_controller_attribute(rebalance); read_attribute(rebalance_status); -rw_attribute(promote_whole_extents); read_attribute(new_stripes); @@ -347,8 +346,6 @@ SHOW(bch2_fs) if (attr == &sysfs_rebalance_status) bch2_rebalance_status_to_text(out, c); - sysfs_print(promote_whole_extents, c->promote_whole_extents); - /* Debugging: */ if (attr == &sysfs_journal_debug) @@ -436,8 +433,6 @@ STORE(bch2_fs) sysfs_pd_controller_store(rebalance, &c->rebalance.pd); - sysfs_strtoul(promote_whole_extents, c->promote_whole_extents); - /* Debugging: */ if (!test_bit(BCH_FS_started, &c->flags)) @@ -514,7 +509,6 @@ struct attribute *bch2_fs_files[] = { &sysfs_btree_cache_size, &sysfs_btree_write_stats, - &sysfs_promote_whole_extents, &sysfs_rebalance_status, &sysfs_compression_stats, From d89b35d83ee398bda2b4294ef7c6a2ab663044cf Mon Sep 17 00:00:00 2001 From: Xiaxi Shen Date: Wed, 7 Aug 2024 00:10:05 -0700 Subject: [PATCH 158/302] bcachefs: Fix a spelling error in docs Signed-off-by: Xiaxi Shen Signed-off-by: Kent Overstreet --- Documentation/filesystems/bcachefs/CodingStyle.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/filesystems/bcachefs/CodingStyle.rst b/Documentation/filesystems/bcachefs/CodingStyle.rst index 0c45829a4899..01de555e21d8 100644 --- a/Documentation/filesystems/bcachefs/CodingStyle.rst +++ b/Documentation/filesystems/bcachefs/CodingStyle.rst @@ -175,7 +175,7 @@ errors in our thinking by running our code and seeing what happens. If your time is being wasted because your tools are bad or too slow - don't accept it, fix it. -Put effort into your documentation, commmit messages, and code comments - but +Put effort into your documentation, commit messages, and code comments - but don't go overboard. A good commit message is wonderful - but if the information was important enough to go in a commit message, ask yourself if it would be even better as a code comment. From 11827dba08e8b909076fe3a40f0a99a6e3d84eef Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 7 Aug 2024 15:44:57 -0400 Subject: [PATCH 159/302] bcachefs: trivial open_bucket_add_buckets() cleanup Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_foreground.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/fs/bcachefs/alloc_foreground.c b/fs/bcachefs/alloc_foreground.c index 8563c2d26847..084b03b8c6e9 100644 --- a/fs/bcachefs/alloc_foreground.c +++ b/fs/bcachefs/alloc_foreground.c @@ -1022,9 +1022,6 @@ static int __open_bucket_add_buckets(struct btree_trans *trans, open_bucket_for_each(c, ptrs, ob, i) __clear_bit(ob->dev, devs.d); - if (erasure_code && ec_open_bucket(c, ptrs)) - return 0; - ret = bucket_alloc_set_writepoint(c, ptrs, wp, &devs, nr_replicas, nr_effective, have_cache, erasure_code, flags); @@ -1079,7 +1076,7 @@ static int open_bucket_add_buckets(struct btree_trans *trans, { int ret; - if (erasure_code) { + if (erasure_code && !ec_open_bucket(trans->c, ptrs)) { ret = __open_bucket_add_buckets(trans, ptrs, wp, devs_have, target, erasure_code, nr_replicas, nr_effective, have_cache, From 59a1a62a4227b4cb5c2e197de4aa1c727f68e4c7 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 8 Aug 2024 11:40:47 -0400 Subject: [PATCH 160/302] bcachefs: bch2_sb_nr_devices() factoring out a helper Signed-off-by: Kent Overstreet --- fs/bcachefs/sb-members.c | 9 +++++++++ fs/bcachefs/sb-members.h | 2 ++ fs/bcachefs/super-io.c | 9 ++------- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/fs/bcachefs/sb-members.c b/fs/bcachefs/sb-members.c index 4b765422dd77..7aa5067e09c5 100644 --- a/fs/bcachefs/sb-members.c +++ b/fs/bcachefs/sb-members.c @@ -465,3 +465,12 @@ void bch2_dev_btree_bitmap_mark(struct bch_fs *c, struct bkey_s_c k) __bch2_dev_btree_bitmap_mark(mi, ptr->dev, ptr->offset, btree_sectors(c)); } } + +unsigned bch2_sb_nr_devices(const struct bch_sb *sb) +{ + unsigned nr = 0; + + for (unsigned i = 0; i < sb->nr_devices; i++) + nr += bch2_member_exists((struct bch_sb *) sb, i); + return nr; +} diff --git a/fs/bcachefs/sb-members.h b/fs/bcachefs/sb-members.h index dd93192ec065..f307f2857603 100644 --- a/fs/bcachefs/sb-members.h +++ b/fs/bcachefs/sb-members.h @@ -307,6 +307,8 @@ static inline bool bch2_member_exists(struct bch_sb *sb, unsigned dev) return false; } +unsigned bch2_sb_nr_devices(const struct bch_sb *); + static inline struct bch_member_cpu bch2_mi_to_cpu(struct bch_member *mi) { return (struct bch_member_cpu) { diff --git a/fs/bcachefs/super-io.c b/fs/bcachefs/super-io.c index 2a72782d6d17..d86d5dae54c9 100644 --- a/fs/bcachefs/super-io.c +++ b/fs/bcachefs/super-io.c @@ -1295,15 +1295,9 @@ void bch2_sb_layout_to_text(struct printbuf *out, struct bch_sb_layout *l) void bch2_sb_to_text(struct printbuf *out, struct bch_sb *sb, bool print_layout, unsigned fields) { - u64 fields_have = 0; - unsigned nr_devices = 0; - if (!out->nr_tabstops) printbuf_tabstop_push(out, 44); - for (int i = 0; i < sb->nr_devices; i++) - nr_devices += bch2_member_exists(sb, i); - prt_printf(out, "External UUID:\t"); pr_uuid(out, sb->user_uuid.b); prt_newline(out); @@ -1359,9 +1353,10 @@ void bch2_sb_to_text(struct printbuf *out, struct bch_sb *sb, prt_newline(out); prt_printf(out, "Clean:\t%llu\n", BCH_SB_CLEAN(sb)); - prt_printf(out, "Devices:\t%u\n", nr_devices); + prt_printf(out, "Devices:\t%u\n", bch2_sb_nr_devices(sb)); prt_printf(out, "Sections:\t"); + u64 fields_have = 0; vstruct_for_each(sb, f) fields_have |= 1 << le32_to_cpu(f->type); prt_bitflags(out, bch2_sb_fields, fields_have); From 2a463e948a31b02bf9bb4f70b5e9ee71ce3f4ce1 Mon Sep 17 00:00:00 2001 From: Alan Huang Date: Sun, 11 Aug 2024 00:52:24 +0800 Subject: [PATCH 161/302] bcachefs: Remove unused parameter of bkey_mantissa The idx parameter of bkey_mantissa became unused since commit b904a7991802 ("bcachefs: Go back to 16 bit mantissa bkey floats"), so remove it. Signed-off-by: Alan Huang Signed-off-by: Kent Overstreet --- fs/bcachefs/bset.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fs/bcachefs/bset.c b/fs/bcachefs/bset.c index 575e1d0b6eeb..61ccb56b26f5 100644 --- a/fs/bcachefs/bset.c +++ b/fs/bcachefs/bset.c @@ -585,8 +585,7 @@ static unsigned rw_aux_tree_bsearch(struct btree *b, } static inline unsigned bkey_mantissa(const struct bkey_packed *k, - const struct bkey_float *f, - unsigned idx) + const struct bkey_float *f) { u64 v; @@ -668,7 +667,7 @@ static __always_inline void make_bfloat(struct btree *b, struct bset_tree *t, EBUG_ON(shift < 0 || shift >= BFLOAT_FAILED); f->exponent = shift; - mantissa = bkey_mantissa(m, f, j); + mantissa = bkey_mantissa(m, f); /* * If we've got garbage bits, set them to all 1s - it's legal for the @@ -1133,7 +1132,7 @@ static struct bkey_packed *bset_search_tree(const struct btree *b, goto slowpath; l = f->mantissa; - r = bkey_mantissa(packed_search, f, n); + r = bkey_mantissa(packed_search, f); if (unlikely(l == r) && bkey_mantissa_bits_dropped(b, f, n)) goto slowpath; From 89ae9a04b2fff507395ef0a6958bfc4f75886f7e Mon Sep 17 00:00:00 2001 From: Alan Huang Date: Sun, 11 Aug 2024 00:52:25 +0800 Subject: [PATCH 162/302] bcachefs: Remove unused parameter of bkey_mantissa_bits_dropped The idx parameter of bkey_mantissa_bits_dropped is unused, remove it. Signed-off-by: Alan Huang Signed-off-by: Kent Overstreet --- fs/bcachefs/bset.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fs/bcachefs/bset.c b/fs/bcachefs/bset.c index 61ccb56b26f5..45e8ab5c3c5c 100644 --- a/fs/bcachefs/bset.c +++ b/fs/bcachefs/bset.c @@ -1097,8 +1097,7 @@ static inline void prefetch_four_cachelines(void *p) } static inline bool bkey_mantissa_bits_dropped(const struct btree *b, - const struct bkey_float *f, - unsigned idx) + const struct bkey_float *f) { #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ unsigned key_bits_start = b->format.key_u64s * 64 - b->nr_key_bits; @@ -1134,7 +1133,7 @@ static struct bkey_packed *bset_search_tree(const struct btree *b, l = f->mantissa; r = bkey_mantissa(packed_search, f); - if (unlikely(l == r) && bkey_mantissa_bits_dropped(b, f, n)) + if (unlikely(l == r) && bkey_mantissa_bits_dropped(b, f)) goto slowpath; n = n * 2 + (l < r); From 6cca8319e017d1732ef4c951952109d9ca8506c0 Mon Sep 17 00:00:00 2001 From: Alan Huang Date: Sat, 10 Aug 2024 23:51:40 +0800 Subject: [PATCH 163/302] bcachefs: Remove dead code in __build_ro_aux_tree This logic is no longer useful since commit 3ce8b463e3e0 ("bcachefs: kill bset_tree->max_key"), so remove it. Signed-off-by: Alan Huang Signed-off-by: Kent Overstreet --- fs/bcachefs/bset.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/fs/bcachefs/bset.c b/fs/bcachefs/bset.c index 45e8ab5c3c5c..59158eb6d41f 100644 --- a/fs/bcachefs/bset.c +++ b/fs/bcachefs/bset.c @@ -753,9 +753,6 @@ retry: EBUG_ON(tree_to_bkey(b, t, j) != k); } - while (k != btree_bkey_last(b, t)) - prev = k, k = bkey_p_next(k); - if (!bkey_pack_pos(bkey_to_packed(&min_key), b->data->min_key, b)) { bkey_init(&min_key.k); min_key.k.p = b->data->min_key; From 288a6690eb51c507a28cb599944096e4d3c42e94 Mon Sep 17 00:00:00 2001 From: Alan Huang Date: Sun, 11 Aug 2024 00:11:46 +0800 Subject: [PATCH 164/302] bcachefs: Convert open-coded extra computation to helper This patch replaces open-coded extra computation to eytzinger1_extra. Signed-off-by: Alan Huang Signed-off-by: Kent Overstreet --- fs/bcachefs/bset.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/bcachefs/bset.c b/fs/bcachefs/bset.c index 59158eb6d41f..00a821f617a5 100644 --- a/fs/bcachefs/bset.c +++ b/fs/bcachefs/bset.c @@ -732,7 +732,7 @@ retry: return; } - t->extra = (t->size - rounddown_pow_of_two(t->size - 1)) << 1; + t->extra = eytzinger1_extra(t->size - 1); /* First we figure out where the first key in each cacheline is */ eytzinger1_for_each(j, t->size - 1) { From 5d01101284a3286ee600e81bde4e4e7e46385ccc Mon Sep 17 00:00:00 2001 From: Alan Huang Date: Mon, 12 Aug 2024 16:06:09 +0800 Subject: [PATCH 165/302] bcachefs: Minimize the search range used to calculate the mantissa When the search key's mantissa is larger than the node i's, we know that the search key is larger than the first key of the cacheline corresponding to node i, so that when we are calculating the mantissa of right side nodes of node i, the left side of the search range can be the first key of node i. Once the search range is minimized, the mantissa we are calculating can have more useful bits, thus reduce the slow path comparison. Besides, we can now remove all the prev array stuff. Signed-off-by: Alan Huang Signed-off-by: Kent Overstreet --- fs/bcachefs/bset.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/bcachefs/bset.c b/fs/bcachefs/bset.c index 00a821f617a5..43b40f6b8c0a 100644 --- a/fs/bcachefs/bset.c +++ b/fs/bcachefs/bset.c @@ -616,7 +616,7 @@ static __always_inline void make_bfloat(struct btree *b, struct bset_tree *t, struct bkey_packed *m = tree_to_bkey(b, t, j); struct bkey_packed *l = is_power_of_2(j) ? min_key - : tree_to_prev_bkey(b, t, j >> ffs(j)); + : tree_to_bkey(b, t, j >> ffs(j)); struct bkey_packed *r = is_power_of_2(j + 1) ? max_key : tree_to_bkey(b, t, j >> (ffz(j) + 1)); From 3130303bd9b7bda1550b880c9843c44a16932feb Mon Sep 17 00:00:00 2001 From: Alan Huang Date: Mon, 12 Aug 2024 17:04:04 +0800 Subject: [PATCH 166/302] bcachefs: Remove the prev array stuff After reducing the search range when building the aux tree, the prev array stuff is no longer useful, so remove it. Signed-off-by: Alan Huang Signed-off-by: Kent Overstreet --- fs/bcachefs/bset.c | 34 ++++------------------------------ 1 file changed, 4 insertions(+), 30 deletions(-) diff --git a/fs/bcachefs/bset.c b/fs/bcachefs/bset.c index 43b40f6b8c0a..0ccc1d962428 100644 --- a/fs/bcachefs/bset.c +++ b/fs/bcachefs/bset.c @@ -304,11 +304,6 @@ struct bkey_float { }; #define BKEY_MANTISSA_BITS 16 -static unsigned bkey_float_byte_offset(unsigned idx) -{ - return idx * sizeof(struct bkey_float); -} - struct ro_aux_tree { u8 nothing[0]; struct bkey_float f[]; @@ -328,8 +323,7 @@ static unsigned bset_aux_tree_buf_end(const struct bset_tree *t) return t->aux_data_offset; case BSET_RO_AUX_TREE: return t->aux_data_offset + - DIV_ROUND_UP(t->size * sizeof(struct bkey_float) + - t->size * sizeof(u8), 8); + DIV_ROUND_UP(t->size * sizeof(struct bkey_float), 8); case BSET_RW_AUX_TREE: return t->aux_data_offset + DIV_ROUND_UP(sizeof(struct rw_aux_tree) * t->size, 8); @@ -360,14 +354,6 @@ static struct ro_aux_tree *ro_aux_tree_base(const struct btree *b, return __aux_tree_base(b, t); } -static u8 *ro_aux_tree_prev(const struct btree *b, - const struct bset_tree *t) -{ - EBUG_ON(bset_aux_tree_type(t) != BSET_RO_AUX_TREE); - - return __aux_tree_base(b, t) + bkey_float_byte_offset(t->size); -} - static struct bkey_float *bkey_float(const struct btree *b, const struct bset_tree *t, unsigned idx) @@ -479,15 +465,6 @@ static inline struct bkey_packed *tree_to_bkey(const struct btree *b, bkey_float(b, t, j)->key_offset); } -static struct bkey_packed *tree_to_prev_bkey(const struct btree *b, - const struct bset_tree *t, - unsigned j) -{ - unsigned prev_u64s = ro_aux_tree_prev(b, t)[j]; - - return (void *) ((u64 *) tree_to_bkey(b, t, j)->_data - prev_u64s); -} - static struct rw_aux_tree *rw_aux_tree(const struct btree *b, const struct bset_tree *t) { @@ -689,8 +666,7 @@ static unsigned __bset_tree_capacity(struct btree *b, const struct bset_tree *t) static unsigned bset_ro_tree_capacity(struct btree *b, const struct bset_tree *t) { - return __bset_tree_capacity(b, t) / - (sizeof(struct bkey_float) + sizeof(u8)); + return __bset_tree_capacity(b, t) / sizeof(struct bkey_float); } static unsigned bset_rw_tree_capacity(struct btree *b, const struct bset_tree *t) @@ -719,7 +695,7 @@ static noinline void __build_rw_aux_tree(struct btree *b, struct bset_tree *t) static noinline void __build_ro_aux_tree(struct btree *b, struct bset_tree *t) { - struct bkey_packed *prev = NULL, *k = btree_bkey_first(b, t); + struct bkey_packed *k = btree_bkey_first(b, t); struct bkey_i min_key, max_key; unsigned cacheline = 1; @@ -737,7 +713,7 @@ retry: /* First we figure out where the first key in each cacheline is */ eytzinger1_for_each(j, t->size - 1) { while (bkey_to_cacheline(b, t, k) < cacheline) - prev = k, k = bkey_p_next(k); + k = bkey_p_next(k); if (k >= btree_bkey_last(b, t)) { /* XXX: this path sucks */ @@ -745,11 +721,9 @@ retry: goto retry; } - ro_aux_tree_prev(b, t)[j] = prev->u64s; bkey_float(b, t, j)->key_offset = bkey_to_cacheline_offset(b, t, cacheline++, k); - EBUG_ON(tree_to_prev_bkey(b, t, j) != prev); EBUG_ON(tree_to_bkey(b, t, j) != k); } From 5935bf3341b82bc76f1d816842273c06615aa727 Mon Sep 17 00:00:00 2001 From: Alan Huang Date: Wed, 14 Aug 2024 22:20:07 +0800 Subject: [PATCH 167/302] bcachefs: Remove unused parameter iter here is unused, remove it. Signed-off-by: Alan Huang Signed-off-by: Kent Overstreet --- fs/bcachefs/bset.c | 1 - fs/bcachefs/bset.h | 4 ++-- fs/bcachefs/btree_trans_commit.c | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/fs/bcachefs/bset.c b/fs/bcachefs/bset.c index 0ccc1d962428..1b66b2c7e018 100644 --- a/fs/bcachefs/bset.c +++ b/fs/bcachefs/bset.c @@ -969,7 +969,6 @@ static void bch2_bset_fix_lookup_table(struct btree *b, } void bch2_bset_insert(struct btree *b, - struct btree_node_iter *iter, struct bkey_packed *where, struct bkey_i *insert, unsigned clobber_u64s) diff --git a/fs/bcachefs/bset.h b/fs/bcachefs/bset.h index 5c6c7a14fa0f..6953d55b72cc 100644 --- a/fs/bcachefs/bset.h +++ b/fs/bcachefs/bset.h @@ -270,8 +270,8 @@ void bch2_bset_init_first(struct btree *, struct bset *); void bch2_bset_init_next(struct btree *, struct btree_node_entry *); void bch2_bset_build_aux_tree(struct btree *, struct bset_tree *, bool); -void bch2_bset_insert(struct btree *, struct btree_node_iter *, - struct bkey_packed *, struct bkey_i *, unsigned); +void bch2_bset_insert(struct btree *, struct bkey_packed *, struct bkey_i *, + unsigned); void bch2_bset_delete(struct btree *, struct bkey_packed *, unsigned); /* Bkey utility code */ diff --git a/fs/bcachefs/btree_trans_commit.c b/fs/bcachefs/btree_trans_commit.c index a0101d9c5d83..91884da4e30a 100644 --- a/fs/bcachefs/btree_trans_commit.c +++ b/fs/bcachefs/btree_trans_commit.c @@ -214,7 +214,7 @@ bool bch2_btree_bset_insert_key(struct btree_trans *trans, k = bch2_btree_node_iter_bset_pos(node_iter, b, bset_tree_last(b)); overwrite: - bch2_bset_insert(b, node_iter, k, insert, clobber_u64s); + bch2_bset_insert(b, k, insert, clobber_u64s); new_u64s = k->u64s; fix_iter: if (clobber_u64s != new_u64s) From d3f30f16294cc8e58ef14f934874ed7b3ec15b71 Mon Sep 17 00:00:00 2001 From: Youling Tang Date: Thu, 15 Aug 2024 16:57:43 +0800 Subject: [PATCH 168/302] bcachefs: drop unused posix acl handlers Remove struct nop_posix_acl_{access,default} for bcachefs filesystem that don't depend on the xattr handler in their inode->i_op->listxattr() method in any way. There's nothing more to do than to simply remove the handler. It's been effectively unused ever since we introduced the new posix acl api. See [1] for details. Link [1]: https://patchwork.kernel.org/project/linux-fsdevel/cover/20230125-fs-acl-remove-generic-xattr-handlers-v3-0-f760cc58967d@kernel.org/ Signed-off-by: Youling Tang Signed-off-by: Kent Overstreet --- fs/bcachefs/xattr.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/fs/bcachefs/xattr.c b/fs/bcachefs/xattr.c index 9f810fb36ba1..7b3d9fdae244 100644 --- a/fs/bcachefs/xattr.c +++ b/fs/bcachefs/xattr.c @@ -601,10 +601,6 @@ static const struct xattr_handler bch_xattr_bcachefs_effective_handler = { const struct xattr_handler *bch2_xattr_handlers[] = { &bch_xattr_user_handler, -#ifdef CONFIG_BCACHEFS_POSIX_ACL - &nop_posix_acl_access, - &nop_posix_acl_default, -#endif &bch_xattr_trusted_handler, &bch_xattr_security_handler, #ifndef NO_BCACHEFS_FS From 42386fbaee1dd41f2c1ad41301a43454e808322c Mon Sep 17 00:00:00 2001 From: Youling Tang Date: Thu, 15 Aug 2024 16:57:44 +0800 Subject: [PATCH 169/302] bcachefs: Simplify bch2_xattr_emit() implementation Use helper functions to make code more readable. Similar to commit a5488f29835c ("fs: simplify ->listxattr() implementation") Signed-off-by: Youling Tang Signed-off-by: Kent Overstreet --- fs/bcachefs/xattr.c | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/fs/bcachefs/xattr.c b/fs/bcachefs/xattr.c index 7b3d9fdae244..56c8d3fe55a4 100644 --- a/fs/bcachefs/xattr.c +++ b/fs/bcachefs/xattr.c @@ -250,17 +250,27 @@ static int __bch2_xattr_emit(const char *prefix, return 0; } +static inline const char *bch2_xattr_prefix(unsigned type, struct dentry *dentry) +{ + const struct xattr_handler *handler = bch2_xattr_type_to_handler(type); + + if (!xattr_handler_can_list(handler, dentry)) + return NULL; + + return xattr_prefix(handler); +} + static int bch2_xattr_emit(struct dentry *dentry, const struct bch_xattr *xattr, struct xattr_buf *buf) { - const struct xattr_handler *handler = - bch2_xattr_type_to_handler(xattr->x_type); + const char *prefix; - return handler && (!handler->list || handler->list(dentry)) - ? __bch2_xattr_emit(handler->prefix ?: handler->name, - xattr->x_name, xattr->x_name_len, buf) - : 0; + prefix = bch2_xattr_prefix(xattr->x_type, dentry); + if (!prefix) + return 0; + + return __bch2_xattr_emit(prefix, xattr->x_name, xattr->x_name_len, buf); } static int bch2_xattr_list_bcachefs(struct bch_fs *c, From b36f679c99889bbe9c48e09f7c175d1058823ae8 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Mon, 19 Aug 2024 15:11:20 -0400 Subject: [PATCH 170/302] bcachefs: Drop memalloc_nofs_save() in bch2_btree_node_mem_alloc() It's really not needed: the only locks used here are the btree cache lock, which we drop for GFP_WAIT allocations, and btree node locks - but we also drop those for GFP_WAIT allocations. Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_cache.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/fs/bcachefs/btree_cache.c b/fs/bcachefs/btree_cache.c index e52a06d3418c..662f0f79b7af 100644 --- a/fs/bcachefs/btree_cache.c +++ b/fs/bcachefs/btree_cache.c @@ -671,9 +671,7 @@ struct btree *bch2_btree_node_mem_alloc(struct btree_trans *trans, bool pcpu_rea : &bc->freed_nonpcpu; struct btree *b, *b2; u64 start_time = local_clock(); - unsigned flags; - flags = memalloc_nofs_save(); mutex_lock(&bc->lock); /* @@ -745,8 +743,6 @@ out: bch2_time_stats_update(&c->times[BCH_TIME_btree_node_mem_alloc], start_time); - memalloc_nofs_restore(flags); - int ret = bch2_trans_relock(trans); if (unlikely(ret)) { bch2_btree_node_to_freelist(c, b); @@ -781,7 +777,6 @@ err: } mutex_unlock(&bc->lock); - memalloc_nofs_restore(flags); return ERR_PTR(-BCH_ERR_ENOMEM_btree_node_mem_alloc); } From fdbc9c390ade238383f5a24e4b32e49550dc90e4 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Mon, 19 Aug 2024 15:33:38 -0400 Subject: [PATCH 171/302] bcachefs: bch2_time_stats_reset() Signed-off-by: Kent Overstreet --- fs/bcachefs/sysfs.c | 9 ++++++++- fs/bcachefs/time_stats.c | 14 ++++++++++++++ fs/bcachefs/time_stats.h | 3 ++- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/fs/bcachefs/sysfs.c b/fs/bcachefs/sysfs.c index 7ff9afcb37b2..6791540d6a4a 100644 --- a/fs/bcachefs/sysfs.c +++ b/fs/bcachefs/sysfs.c @@ -233,7 +233,7 @@ write_attribute(perf_test); #define x(_name) \ static struct attribute sysfs_time_stat_##_name = \ - { .name = #_name, .mode = 0444 }; + { .name = #_name, .mode = 0644 }; BCH_TIME_STATS() #undef x @@ -722,6 +722,13 @@ SHOW(bch2_fs_time_stats) STORE(bch2_fs_time_stats) { + struct bch_fs *c = container_of(kobj, struct bch_fs, time_stats); + +#define x(name) \ + if (attr == &sysfs_time_stat_##name) \ + bch2_time_stats_reset(&c->times[BCH_TIME_##name]); + BCH_TIME_STATS() +#undef x return size; } SYSFS_OPS(bch2_fs_time_stats); diff --git a/fs/bcachefs/time_stats.c b/fs/bcachefs/time_stats.c index 4508e9dcbee2..3fe82757f93a 100644 --- a/fs/bcachefs/time_stats.c +++ b/fs/bcachefs/time_stats.c @@ -151,6 +151,20 @@ void __bch2_time_stats_update(struct bch2_time_stats *stats, u64 start, u64 end) } } +void bch2_time_stats_reset(struct bch2_time_stats *stats) +{ + spin_lock_irq(&stats->lock); + unsigned offset = offsetof(struct bch2_time_stats, min_duration); + memset((void *) stats + offset, 0, sizeof(*stats) - offset); + + if (stats->buffer) { + int cpu; + for_each_possible_cpu(cpu) + per_cpu_ptr(stats->buffer, cpu)->nr = 0; + } + spin_unlock_irq(&stats->lock); +} + void bch2_time_stats_exit(struct bch2_time_stats *stats) { free_percpu(stats->buffer); diff --git a/fs/bcachefs/time_stats.h b/fs/bcachefs/time_stats.h index 5df61403744b..dc6493f7bbab 100644 --- a/fs/bcachefs/time_stats.h +++ b/fs/bcachefs/time_stats.h @@ -70,6 +70,7 @@ struct time_stat_buffer { struct bch2_time_stats { spinlock_t lock; bool have_quantiles; + struct time_stat_buffer __percpu *buffer; /* all fields are in nanoseconds */ u64 min_duration; u64 max_duration; @@ -87,7 +88,6 @@ struct bch2_time_stats { struct mean_and_variance_weighted duration_stats_weighted; struct mean_and_variance_weighted freq_stats_weighted; - struct time_stat_buffer __percpu *buffer; }; struct bch2_time_stats_quantiles { @@ -142,6 +142,7 @@ static inline bool track_event_change(struct bch2_time_stats *stats, bool v) return false; } +void bch2_time_stats_reset(struct bch2_time_stats *); void bch2_time_stats_exit(struct bch2_time_stats *); void bch2_time_stats_init(struct bch2_time_stats *); From a8cdf0ff46643305be6ae2d580d140bb3832af39 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 20 Aug 2024 05:10:11 +0100 Subject: [PATCH 172/302] bcachefs: Do not check folio_has_private() folio_has_private() is an attractive nuisance; filesystem authors generally don't realise that it actually checks two flags (one of which is never set by bcachefs). There's no need to check the private flag at all; for folios owned by bcachefs, we know that folio->private is NULL when the private flag is clear and non-NULL when the private flag is set. Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Kent Overstreet --- fs/bcachefs/fs-io-pagecache.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fs/bcachefs/fs-io-pagecache.h b/fs/bcachefs/fs-io-pagecache.h index fd7d692c087e..fad911cf5068 100644 --- a/fs/bcachefs/fs-io-pagecache.h +++ b/fs/bcachefs/fs-io-pagecache.h @@ -99,9 +99,7 @@ static inline void bch2_folio_release(struct folio *folio) static inline struct bch_folio *__bch2_folio(struct folio *folio) { - return folio_has_private(folio) - ? (struct bch_folio *) folio_get_private(folio) - : NULL; + return folio_get_private(folio); } static inline struct bch_folio *bch2_folio(struct folio *folio) From f1625637b85191a0ac6f9ebf2b30d8b335f08547 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 30 Jun 2024 09:25:56 -0400 Subject: [PATCH 173/302] bcachefs: Assert that we don't lock nodes when !trans->locked We rely on the trans->locked to know if a trans has nodes locked for assertions about deadlocks; there can't be more than one trans in the same process that is locked. Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_locking.h | 7 +++---- fs/bcachefs/btree_update_interior.c | 24 ++++++++++++------------ fs/bcachefs/btree_update_interior.h | 2 ++ 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/fs/bcachefs/btree_locking.h b/fs/bcachefs/btree_locking.h index dd0a2a1aa2bc..7c07f9fa9add 100644 --- a/fs/bcachefs/btree_locking.h +++ b/fs/bcachefs/btree_locking.h @@ -218,14 +218,12 @@ static inline int __btree_node_lock_nopath(struct btree_trans *trans, bool lock_may_not_fail, unsigned long ip) { - int ret; - trans->lock_may_not_fail = lock_may_not_fail; trans->lock_must_abort = false; trans->locking = b; - ret = six_lock_ip_waiter(&b->lock, type, &trans->locking_wait, - bch2_six_check_for_deadlock, trans, ip); + int ret = six_lock_ip_waiter(&b->lock, type, &trans->locking_wait, + bch2_six_check_for_deadlock, trans, ip); WRITE_ONCE(trans->locking, NULL); WRITE_ONCE(trans->locking_wait.start_time, 0); @@ -284,6 +282,7 @@ static inline int btree_node_lock(struct btree_trans *trans, int ret = 0; EBUG_ON(level >= BTREE_MAX_DEPTH); + bch2_trans_verify_not_unlocked(trans); if (likely(six_trylock_type(&b->lock, type)) || btree_node_lock_increment(trans, b, level, (enum btree_node_locked_type) type) || diff --git a/fs/bcachefs/btree_update_interior.c b/fs/bcachefs/btree_update_interior.c index 396d052474bb..1433aefb47fd 100644 --- a/fs/bcachefs/btree_update_interior.c +++ b/fs/bcachefs/btree_update_interior.c @@ -731,6 +731,18 @@ static void btree_update_nodes_written(struct btree_update *as) bch2_fs_fatal_err_on(ret && !bch2_journal_error(&c->journal), c, "%s", bch2_err_str(ret)); err: + /* + * Ensure transaction is unlocked before using btree_node_lock_nopath() + * (the use of which is always suspect, we need to work on removing this + * in the future) + * + * It should be, but bch2_path_get_unlocked_mut() -> bch2_path_get() + * calls bch2_path_upgrade(), before we call path_make_mut(), so we may + * rarely end up with a locked path besides the one we have here: + */ + bch2_trans_unlock(trans); + bch2_trans_begin(trans); + /* * We have to be careful because another thread might be getting ready * to free as->b and calling btree_update_reparent() on us - we'll @@ -750,18 +762,6 @@ err: * we're in journal error state: */ - /* - * Ensure transaction is unlocked before using - * btree_node_lock_nopath() (the use of which is always suspect, - * we need to work on removing this in the future) - * - * It should be, but bch2_path_get_unlocked_mut() -> bch2_path_get() - * calls bch2_path_upgrade(), before we call path_make_mut(), so - * we may rarely end up with a locked path besides the one we - * have here: - */ - bch2_trans_unlock(trans); - bch2_trans_begin(trans); btree_path_idx_t path_idx = bch2_path_get_unlocked_mut(trans, as->btree_id, b->c.level, b->key.k.p); struct btree_path *path = trans->paths + path_idx; diff --git a/fs/bcachefs/btree_update_interior.h b/fs/bcachefs/btree_update_interior.h index 02c6ecada97c..10f400957f21 100644 --- a/fs/bcachefs/btree_update_interior.h +++ b/fs/bcachefs/btree_update_interior.h @@ -159,6 +159,8 @@ static inline int bch2_foreground_maybe_merge(struct btree_trans *trans, unsigned level, unsigned flags) { + bch2_trans_verify_not_unlocked(trans); + return bch2_foreground_maybe_merge_sibling(trans, path, level, flags, btree_prev_sib) ?: bch2_foreground_maybe_merge_sibling(trans, path, level, flags, From 94932a0842ccebe413cd8205632a537f275c100d Mon Sep 17 00:00:00 2001 From: Alan Huang Date: Thu, 15 Aug 2024 23:40:53 +0800 Subject: [PATCH 174/302] bcachefs: Refactor bch2_bset_fix_lookup_table bch2_bset_fix_lookup_table is too complicated to be easily understood, the comment "l now > where" there is also incorrect when where == t->end_offset. This patch therefore refactor the function, the idea is that when where >= rw_aux_tree(b, t)[t->size - 1].offset, we don't need to adjust the rw aux tree. Signed-off-by: Alan Huang Signed-off-by: Kent Overstreet --- fs/bcachefs/bset.c | 130 ++++++++++++++++++++++++--------------------- 1 file changed, 69 insertions(+), 61 deletions(-) diff --git a/fs/bcachefs/bset.c b/fs/bcachefs/bset.c index 1b66b2c7e018..d1f6092624d8 100644 --- a/fs/bcachefs/bset.c +++ b/fs/bcachefs/bset.c @@ -885,66 +885,18 @@ struct bkey_packed *bch2_bkey_prev_filter(struct btree *b, /* Insert */ -static void bch2_bset_fix_lookup_table(struct btree *b, - struct bset_tree *t, - struct bkey_packed *_where, - unsigned clobber_u64s, - unsigned new_u64s) +static void rw_aux_tree_insert_entry(struct btree *b, + struct bset_tree *t, + unsigned idx) { - int shift = new_u64s - clobber_u64s; - unsigned l, j, where = __btree_node_key_to_offset(b, _where); - - EBUG_ON(bset_has_ro_aux_tree(t)); - - if (!bset_has_rw_aux_tree(t)) - return; - - /* returns first entry >= where */ - l = rw_aux_tree_bsearch(b, t, where); - - if (!l) /* never delete first entry */ - l++; - else if (l < t->size && - where < t->end_offset && - rw_aux_tree(b, t)[l].offset == where) - rw_aux_tree_set(b, t, l++, _where); - - /* l now > where */ - - for (j = l; - j < t->size && - rw_aux_tree(b, t)[j].offset < where + clobber_u64s; - j++) - ; - - if (j < t->size && - rw_aux_tree(b, t)[j].offset + shift == - rw_aux_tree(b, t)[l - 1].offset) - j++; - - memmove(&rw_aux_tree(b, t)[l], - &rw_aux_tree(b, t)[j], - (void *) &rw_aux_tree(b, t)[t->size] - - (void *) &rw_aux_tree(b, t)[j]); - t->size -= j - l; - - for (j = l; j < t->size; j++) - rw_aux_tree(b, t)[j].offset += shift; - - EBUG_ON(l < t->size && - rw_aux_tree(b, t)[l].offset == - rw_aux_tree(b, t)[l - 1].offset); + EBUG_ON(!idx || idx > t->size); + struct bkey_packed *start = rw_aux_to_bkey(b, t, idx - 1); + struct bkey_packed *end = idx < t->size + ? rw_aux_to_bkey(b, t, idx) + : btree_bkey_last(b, t); if (t->size < bset_rw_tree_capacity(b, t) && - (l < t->size - ? rw_aux_tree(b, t)[l].offset - : t->end_offset) - - rw_aux_tree(b, t)[l - 1].offset > - L1_CACHE_BYTES / sizeof(u64)) { - struct bkey_packed *start = rw_aux_to_bkey(b, t, l - 1); - struct bkey_packed *end = l < t->size - ? rw_aux_to_bkey(b, t, l) - : btree_bkey_last(b, t); + (void *) end - (void *) start > L1_CACHE_BYTES) { struct bkey_packed *k = start; while (1) { @@ -953,17 +905,73 @@ static void bch2_bset_fix_lookup_table(struct btree *b, break; if ((void *) k - (void *) start >= L1_CACHE_BYTES) { - memmove(&rw_aux_tree(b, t)[l + 1], - &rw_aux_tree(b, t)[l], + memmove(&rw_aux_tree(b, t)[idx + 1], + &rw_aux_tree(b, t)[idx], (void *) &rw_aux_tree(b, t)[t->size] - - (void *) &rw_aux_tree(b, t)[l]); + (void *) &rw_aux_tree(b, t)[idx]); t->size++; - rw_aux_tree_set(b, t, l, k); + rw_aux_tree_set(b, t, idx, k); break; } } } +} +static void bch2_bset_fix_lookup_table(struct btree *b, + struct bset_tree *t, + struct bkey_packed *_where, + unsigned clobber_u64s, + unsigned new_u64s) +{ + int shift = new_u64s - clobber_u64s; + unsigned idx, j, where = __btree_node_key_to_offset(b, _where); + + EBUG_ON(bset_has_ro_aux_tree(t)); + + if (!bset_has_rw_aux_tree(t)) + return; + + if (where > rw_aux_tree(b, t)[t->size - 1].offset) { + rw_aux_tree_insert_entry(b, t, t->size); + goto verify; + } + + /* returns first entry >= where */ + idx = rw_aux_tree_bsearch(b, t, where); + + if (rw_aux_tree(b, t)[idx].offset == where) { + if (!idx) { /* never delete first entry */ + idx++; + } else if (where < t->end_offset) { + rw_aux_tree_set(b, t, idx++, _where); + } else { + EBUG_ON(where != t->end_offset); + rw_aux_tree_insert_entry(b, t, --t->size); + goto verify; + } + } + + EBUG_ON(idx < t->size && rw_aux_tree(b, t)[idx].offset <= where); + if (idx < t->size && + rw_aux_tree(b, t)[idx].offset + shift == + rw_aux_tree(b, t)[idx - 1].offset) { + memmove(&rw_aux_tree(b, t)[idx], + &rw_aux_tree(b, t)[idx + 1], + (void *) &rw_aux_tree(b, t)[t->size] - + (void *) &rw_aux_tree(b, t)[idx + 1]); + t->size -= 1; + } + + for (j = idx; j < t->size; j++) + rw_aux_tree(b, t)[j].offset += shift; + + EBUG_ON(idx < t->size && + rw_aux_tree(b, t)[idx].offset == + rw_aux_tree(b, t)[idx - 1].offset); + + rw_aux_tree_insert_entry(b, t, idx); + +verify: bch2_bset_verify_rw_aux_tree(b, t); bset_aux_tree_verify(b); } From 848c3ff8826b68a587f84f1b00556ab8af651bef Mon Sep 17 00:00:00 2001 From: Chen Yufan Date: Thu, 22 Aug 2024 10:57:31 +0800 Subject: [PATCH 175/302] bcachefs: Convert to use jiffies macros Use jiffies macros instead of using jiffies directly to handle wraparound. Signed-off-by: Chen Yufan Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_background.c | 3 ++- fs/bcachefs/journal_io.c | 3 ++- fs/bcachefs/thread_with_file.c | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c index dc3a4024aab6..a6b9fe7038f3 100644 --- a/fs/bcachefs/alloc_background.c +++ b/fs/bcachefs/alloc_background.c @@ -30,6 +30,7 @@ #include #include #include +#include static void bch2_discard_one_bucket_fast(struct bch_dev *, u64); @@ -2183,7 +2184,7 @@ int bch2_dev_freespace_init(struct bch_fs *c, struct bch_dev *ca, * freespace/need_discard/need_gc_gens btrees as needed: */ while (1) { - if (last_updated + HZ * 10 < jiffies) { + if (time_after(jiffies, last_updated + HZ * 10)) { bch_info(ca, "%s: currently at %llu/%llu", __func__, iter.pos.offset, ca->mi.nbuckets); last_updated = jiffies; diff --git a/fs/bcachefs/journal_io.c b/fs/bcachefs/journal_io.c index 7664b68e6a15..32b886feb2ca 100644 --- a/fs/bcachefs/journal_io.c +++ b/fs/bcachefs/journal_io.c @@ -1950,7 +1950,8 @@ static int bch2_journal_write_pick_flush(struct journal *j, struct journal_buf * if (error || w->noflush || (!w->must_flush && - (jiffies - j->last_flush_write) < msecs_to_jiffies(c->opts.journal_flush_delay) && + time_before(jiffies, j->last_flush_write + + msecs_to_jiffies(c->opts.journal_flush_delay)) && test_bit(JOURNAL_may_skip_flush, &j->flags))) { w->noflush = true; SET_JSET_NO_FLUSH(w->data, true); diff --git a/fs/bcachefs/thread_with_file.c b/fs/bcachefs/thread_with_file.c index 0807ce9b171a..fb3442a7c67f 100644 --- a/fs/bcachefs/thread_with_file.c +++ b/fs/bcachefs/thread_with_file.c @@ -387,7 +387,7 @@ again: seen = buf->buf.nr; char *n = memchr(buf->buf.data, '\n', seen); - if (!n && timeout != MAX_SCHEDULE_TIMEOUT && jiffies >= until) { + if (!n && timeout != MAX_SCHEDULE_TIMEOUT && time_after_eq(jiffies, until)) { spin_unlock(&buf->lock); return -ETIME; } From a803fa551d53e4504a8dfca57817319f71030e2c Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 22 Aug 2024 03:50:22 -0400 Subject: [PATCH 176/302] bcachefs: darray: convert to alloc_hooks() better memory allocation profiling support Signed-off-by: Kent Overstreet --- fs/bcachefs/darray.c | 4 ++-- fs/bcachefs/darray.h | 24 +++++++++--------------- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/fs/bcachefs/darray.c b/fs/bcachefs/darray.c index b7d223f85873..4f06cd8bbbe1 100644 --- a/fs/bcachefs/darray.c +++ b/fs/bcachefs/darray.c @@ -4,12 +4,12 @@ #include #include "darray.h" -int __bch2_darray_resize(darray_char *d, size_t element_size, size_t new_size, gfp_t gfp) +int __bch2_darray_resize_noprof(darray_char *d, size_t element_size, size_t new_size, gfp_t gfp) { if (new_size > d->size) { new_size = roundup_pow_of_two(new_size); - void *data = kvmalloc_array(new_size, element_size, gfp); + void *data = kvmalloc_array_noprof(new_size, element_size, gfp); if (!data) return -ENOMEM; diff --git a/fs/bcachefs/darray.h b/fs/bcachefs/darray.h index 4b340d13caac..8f4c3f0665c4 100644 --- a/fs/bcachefs/darray.h +++ b/fs/bcachefs/darray.h @@ -22,29 +22,23 @@ struct { \ typedef DARRAY(char) darray_char; typedef DARRAY(char *) darray_str; -int __bch2_darray_resize(darray_char *, size_t, size_t, gfp_t); +int __bch2_darray_resize_noprof(darray_char *, size_t, size_t, gfp_t); -static inline int __darray_resize(darray_char *d, size_t element_size, - size_t new_size, gfp_t gfp) -{ - return unlikely(new_size > d->size) - ? __bch2_darray_resize(d, element_size, new_size, gfp) - : 0; -} +#define __bch2_darray_resize(...) alloc_hooks(__bch2_darray_resize_noprof(__VA_ARGS__)) + +#define __darray_resize(_d, _element_size, _new_size, _gfp) \ + (unlikely((_new_size) > (_d)->size) \ + ? __bch2_darray_resize((_d), (_element_size), (_new_size), (_gfp))\ + : 0) #define darray_resize_gfp(_d, _new_size, _gfp) \ - unlikely(__darray_resize((darray_char *) (_d), sizeof((_d)->data[0]), (_new_size), _gfp)) + __darray_resize((darray_char *) (_d), sizeof((_d)->data[0]), (_new_size), _gfp) #define darray_resize(_d, _new_size) \ darray_resize_gfp(_d, _new_size, GFP_KERNEL) -static inline int __darray_make_room(darray_char *d, size_t t_size, size_t more, gfp_t gfp) -{ - return __darray_resize(d, t_size, d->nr + more, gfp); -} - #define darray_make_room_gfp(_d, _more, _gfp) \ - __darray_make_room((darray_char *) (_d), sizeof((_d)->data[0]), (_more), _gfp) + darray_resize_gfp((_d), (_d)->nr + (_more), _gfp) #define darray_make_room(_d, _more) \ darray_make_room_gfp(_d, _more, GFP_KERNEL) From 2c6a7bff2a2eb2d7f183fbe1e9800ae278f415c7 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sat, 24 Aug 2024 11:38:21 -0400 Subject: [PATCH 177/302] bcachefs: Switch gc bucket array to a genradix A user with a 30 tb device is overflowing the INT_MAX limit on vmalloc allocations... Signed-off-by: Kent Overstreet --- fs/bcachefs/bcachefs.h | 2 +- fs/bcachefs/btree_gc.c | 18 ++++-------------- fs/bcachefs/buckets.h | 15 +-------------- fs/bcachefs/buckets_types.h | 8 -------- 4 files changed, 6 insertions(+), 37 deletions(-) diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h index a4773b034f91..ad4df0ad26eb 100644 --- a/fs/bcachefs/bcachefs.h +++ b/fs/bcachefs/bcachefs.h @@ -542,7 +542,7 @@ struct bch_dev { * gc_gens_lock, for device resize - holding any is sufficient for * access: Or rcu_read_lock(), but only for dev_ptr_stale(): */ - struct bucket_array __rcu *buckets_gc; + GENRADIX(struct bucket) buckets_gc; struct bucket_gens __rcu *bucket_gens; u8 *oldest_gen; unsigned long *buckets_nouse; diff --git a/fs/bcachefs/btree_gc.c b/fs/bcachefs/btree_gc.c index eb3002c4eae7..120ffd68ab0a 100644 --- a/fs/bcachefs/btree_gc.c +++ b/fs/bcachefs/btree_gc.c @@ -753,10 +753,8 @@ static void bch2_gc_free(struct bch_fs *c) genradix_free(&c->reflink_gc_table); genradix_free(&c->gc_stripes); - for_each_member_device(c, ca) { - kvfree(rcu_dereference_protected(ca->buckets_gc, 1)); - ca->buckets_gc = NULL; - } + for_each_member_device(c, ca) + genradix_free(&ca->buckets_gc); } static int bch2_gc_start(struct bch_fs *c) @@ -910,20 +908,12 @@ static int bch2_gc_alloc_start(struct bch_fs *c) int ret = 0; for_each_member_device(c, ca) { - struct bucket_array *buckets = kvmalloc(sizeof(struct bucket_array) + - ca->mi.nbuckets * sizeof(struct bucket), - GFP_KERNEL|__GFP_ZERO); - if (!buckets) { + ret = genradix_prealloc(&ca->buckets_gc, ca->mi.nbuckets, GFP_KERNEL); + if (ret) { bch2_dev_put(ca); ret = -BCH_ERR_ENOMEM_gc_alloc_start; break; } - - buckets->first_bucket = ca->mi.first_bucket; - buckets->nbuckets = ca->mi.nbuckets; - buckets->nbuckets_minus_first = - buckets->nbuckets - buckets->first_bucket; - rcu_assign_pointer(ca->buckets_gc, buckets); } bch_err_fn(c, ret); diff --git a/fs/bcachefs/buckets.h b/fs/bcachefs/buckets.h index edbdffd508fc..e2cb7b24b220 100644 --- a/fs/bcachefs/buckets.h +++ b/fs/bcachefs/buckets.h @@ -80,22 +80,9 @@ static inline void bucket_lock(struct bucket *b) TASK_UNINTERRUPTIBLE); } -static inline struct bucket_array *gc_bucket_array(struct bch_dev *ca) -{ - return rcu_dereference_check(ca->buckets_gc, - !ca->fs || - percpu_rwsem_is_held(&ca->fs->mark_lock) || - lockdep_is_held(&ca->fs->state_lock) || - lockdep_is_held(&ca->bucket_lock)); -} - static inline struct bucket *gc_bucket(struct bch_dev *ca, size_t b) { - struct bucket_array *buckets = gc_bucket_array(ca); - - if (b - buckets->first_bucket >= buckets->nbuckets_minus_first) - return NULL; - return buckets->b + b; + return genradix_ptr(&ca->buckets_gc, b); } static inline struct bucket_gens *bucket_gens(struct bch_dev *ca) diff --git a/fs/bcachefs/buckets_types.h b/fs/bcachefs/buckets_types.h index a19460a1bb7c..28bd09a253c8 100644 --- a/fs/bcachefs/buckets_types.h +++ b/fs/bcachefs/buckets_types.h @@ -19,14 +19,6 @@ struct bucket { u32 stripe_sectors; } __aligned(sizeof(long)); -struct bucket_array { - struct rcu_head rcu; - u16 first_bucket; - size_t nbuckets; - size_t nbuckets_minus_first; - struct bucket b[] __counted_by(nbuckets); -}; - struct bucket_gens { struct rcu_head rcu; u16 first_bucket; From 86e92eeeb23741a072fe7532db663250ff2e726a Mon Sep 17 00:00:00 2001 From: Thorsten Blum Date: Sat, 24 Aug 2024 15:57:41 +0200 Subject: [PATCH 178/302] bcachefs: Annotate struct bch_xattr with __counted_by() Add the __counted_by compiler attribute to the flexible array member x_name to improve access bounds-checking via CONFIG_UBSAN_BOUNDS and CONFIG_FORTIFY_SOURCE. Signed-off-by: Thorsten Blum Signed-off-by: Kent Overstreet --- fs/bcachefs/xattr_format.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/bcachefs/xattr_format.h b/fs/bcachefs/xattr_format.h index e9f810539552..c7916011ef34 100644 --- a/fs/bcachefs/xattr_format.h +++ b/fs/bcachefs/xattr_format.h @@ -13,7 +13,7 @@ struct bch_xattr { __u8 x_type; __u8 x_name_len; __le16 x_val_len; - __u8 x_name[]; + __u8 x_name[] __counted_by(x_name_len); } __packed __aligned(8); #endif /* _BCACHEFS_XATTR_FORMAT_H */ From c24adfa0dfc2754f11d91576eabe188671c97209 Mon Sep 17 00:00:00 2001 From: Hongbo Li Date: Sat, 24 Aug 2024 09:27:24 +0800 Subject: [PATCH 179/302] bcachefs: support idmap mounts We enable idmapped mounts for bcachefs. Here, we just pass down the user_namespace argument from the VFS methods to the relevant helpers. The idmap test in bcachefs is as following: ``` 1. losetup /dev/loop1 bcachefs.img 2. ./bcachefs format /dev/loop1 3. mount -t bcachefs /dev/loop1 /mnt/bcachefs/ 4. ./mount-idmapped --map-mount b:0:1000:1 /mnt/bcachefs /mnt/idmapped1/ ll /mnt/bcachefs total 2 drwx------. 2 root root 0 Jun 14 14:10 lost+found -rw-r--r--. 1 root root 1945 Jun 14 14:12 profile ll /mnt/idmapped1/ total 2 drwx------. 2 1000 1000 0 Jun 14 14:10 lost+found -rw-r--r--. 1 1000 1000 1945 Jun 14 14:12 profile Signed-off-by: Hongbo Li Reviewed-by: Christian Brauner Signed-off-by: Kent Overstreet --- fs/bcachefs/fs.c | 50 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/fs/bcachefs/fs.c b/fs/bcachefs/fs.c index 2de776d31b52..866ce1f1a6d1 100644 --- a/fs/bcachefs/fs.c +++ b/fs/bcachefs/fs.c @@ -385,6 +385,8 @@ __bch2_create(struct mnt_idmap *idmap, subvol_inum inum; struct bch_subvolume subvol; u64 journal_seq = 0; + kuid_t kuid; + kgid_t kgid; int ret; /* @@ -411,13 +413,15 @@ __bch2_create(struct mnt_idmap *idmap, retry: bch2_trans_begin(trans); + kuid = mapped_fsuid(idmap, i_user_ns(&dir->v)); + kgid = mapped_fsgid(idmap, i_user_ns(&dir->v)); ret = bch2_subvol_is_ro_trans(trans, dir->ei_inum.subvol) ?: bch2_create_trans(trans, inode_inum(dir), &dir_u, &inode_u, !(flags & BCH_CREATE_TMPFILE) ? &dentry->d_name : NULL, - from_kuid(i_user_ns(&dir->v), current_fsuid()), - from_kgid(i_user_ns(&dir->v), current_fsgid()), + from_kuid(i_user_ns(&dir->v), kuid), + from_kgid(i_user_ns(&dir->v), kgid), mode, rdev, default_acl, acl, snapshot_src, flags) ?: bch2_quota_acct(c, bch_qid(&inode_u), Q_INO, 1, @@ -840,11 +844,17 @@ static void bch2_setattr_copy(struct mnt_idmap *idmap, { struct bch_fs *c = inode->v.i_sb->s_fs_info; unsigned int ia_valid = attr->ia_valid; + kuid_t kuid; + kgid_t kgid; - if (ia_valid & ATTR_UID) - bi->bi_uid = from_kuid(i_user_ns(&inode->v), attr->ia_uid); - if (ia_valid & ATTR_GID) - bi->bi_gid = from_kgid(i_user_ns(&inode->v), attr->ia_gid); + if (ia_valid & ATTR_UID) { + kuid = from_vfsuid(idmap, i_user_ns(&inode->v), attr->ia_vfsuid); + bi->bi_uid = from_kuid(i_user_ns(&inode->v), kuid); + } + if (ia_valid & ATTR_GID) { + kgid = from_vfsgid(idmap, i_user_ns(&inode->v), attr->ia_vfsgid); + bi->bi_gid = from_kgid(i_user_ns(&inode->v), kgid); + } if (ia_valid & ATTR_SIZE) bi->bi_size = attr->ia_size; @@ -859,11 +869,11 @@ static void bch2_setattr_copy(struct mnt_idmap *idmap, if (ia_valid & ATTR_MODE) { umode_t mode = attr->ia_mode; kgid_t gid = ia_valid & ATTR_GID - ? attr->ia_gid + ? kgid : inode->v.i_gid; - if (!in_group_p(gid) && - !capable_wrt_inode_uidgid(idmap, &inode->v, CAP_FSETID)) + if (!in_group_or_capable(idmap, &inode->v, + make_vfsgid(idmap, i_user_ns(&inode->v), gid))) mode &= ~S_ISGID; bi->bi_mode = mode; } @@ -879,17 +889,23 @@ int bch2_setattr_nonsize(struct mnt_idmap *idmap, struct btree_iter inode_iter = { NULL }; struct bch_inode_unpacked inode_u; struct posix_acl *acl = NULL; + kuid_t kuid; + kgid_t kgid; int ret; mutex_lock(&inode->ei_update_lock); qid = inode->ei_qid; - if (attr->ia_valid & ATTR_UID) - qid.q[QTYP_USR] = from_kuid(i_user_ns(&inode->v), attr->ia_uid); + if (attr->ia_valid & ATTR_UID) { + kuid = from_vfsuid(idmap, i_user_ns(&inode->v), attr->ia_vfsuid); + qid.q[QTYP_USR] = from_kuid(i_user_ns(&inode->v), kuid); + } - if (attr->ia_valid & ATTR_GID) - qid.q[QTYP_GRP] = from_kgid(i_user_ns(&inode->v), attr->ia_gid); + if (attr->ia_valid & ATTR_GID) { + kgid = from_vfsgid(idmap, i_user_ns(&inode->v), attr->ia_vfsgid); + qid.q[QTYP_GRP] = from_kgid(i_user_ns(&inode->v), kgid); + } ret = bch2_fs_quota_transfer(c, inode, qid, ~0, KEY_TYPE_QUOTA_PREALLOC); @@ -945,13 +961,15 @@ static int bch2_getattr(struct mnt_idmap *idmap, { struct bch_inode_info *inode = to_bch_ei(d_inode(path->dentry)); struct bch_fs *c = inode->v.i_sb->s_fs_info; + vfsuid_t vfsuid = i_uid_into_vfsuid(idmap, &inode->v); + vfsgid_t vfsgid = i_gid_into_vfsgid(idmap, &inode->v); stat->dev = inode->v.i_sb->s_dev; stat->ino = inode->v.i_ino; stat->mode = inode->v.i_mode; stat->nlink = inode->v.i_nlink; - stat->uid = inode->v.i_uid; - stat->gid = inode->v.i_gid; + stat->uid = vfsuid_into_kuid(vfsuid); + stat->gid = vfsgid_into_kgid(vfsgid); stat->rdev = inode->v.i_rdev; stat->size = i_size_read(&inode->v); stat->atime = inode_get_atime(&inode->v); @@ -2225,7 +2243,7 @@ static struct file_system_type bcache_fs_type = { .name = "bcachefs", .init_fs_context = bch2_init_fs_context, .kill_sb = bch2_kill_sb, - .fs_flags = FS_REQUIRES_DEV, + .fs_flags = FS_REQUIRES_DEV | FS_ALLOW_IDMAP, }; MODULE_ALIAS_FS("bcachefs"); From fa1ab1b46608f6fdc155203c6e4aa7f3da1db434 Mon Sep 17 00:00:00 2001 From: Thorsten Blum Date: Mon, 26 Aug 2024 12:11:36 +0200 Subject: [PATCH 180/302] bcachefs: Annotate bch_replicas_entry_{v0,v1} with __counted_by() Add the __counted_by compiler attribute to the flexible array members devs to improve access bounds-checking via CONFIG_UBSAN_BOUNDS and CONFIG_FORTIFY_SOURCE. Increment nr_devs before adding a new device to the devs array and adjust the array indexes accordingly. Add a helper macro for adding a new device. In bch2_journal_read(), explicitly set nr_devs to 0. Signed-off-by: Thorsten Blum Signed-off-by: Kent Overstreet --- fs/bcachefs/buckets.c | 2 +- fs/bcachefs/journal_io.c | 3 ++- fs/bcachefs/replicas.c | 6 +++--- fs/bcachefs/replicas_format.h | 9 +++++++-- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c index 721bbe1dffc1..4a0f55dc78c4 100644 --- a/fs/bcachefs/buckets.c +++ b/fs/bcachefs/buckets.c @@ -741,7 +741,7 @@ static int __trigger_extent(struct btree_trans *trans, return ret; } else if (!p.has_ec) { *replicas_sectors += disk_sectors; - acc_replicas_key.replicas.devs[acc_replicas_key.replicas.nr_devs++] = p.ptr.dev; + replicas_entry_add_dev(&acc_replicas_key.replicas, p.ptr.dev); } else { ret = bch2_trigger_stripe_ptr(trans, k, p, data_type, disk_sectors, flags); if (ret) diff --git a/fs/bcachefs/journal_io.c b/fs/bcachefs/journal_io.c index 32b886feb2ca..30460bce04be 100644 --- a/fs/bcachefs/journal_io.c +++ b/fs/bcachefs/journal_io.c @@ -1353,6 +1353,7 @@ int bch2_journal_read(struct bch_fs *c, genradix_for_each(&c->journal_entries, radix_iter, _i) { struct bch_replicas_padded replicas = { .e.data_type = BCH_DATA_journal, + .e.nr_devs = 0, .e.nr_required = 1, }; @@ -1379,7 +1380,7 @@ int bch2_journal_read(struct bch_fs *c, goto err; darray_for_each(i->ptrs, ptr) - replicas.e.devs[replicas.e.nr_devs++] = ptr->dev; + replicas_entry_add_dev(&replicas.e, ptr->dev); bch2_replicas_entry_sort(&replicas.e); diff --git a/fs/bcachefs/replicas.c b/fs/bcachefs/replicas.c index 1f34c92a6d11..659ee0516c25 100644 --- a/fs/bcachefs/replicas.c +++ b/fs/bcachefs/replicas.c @@ -123,7 +123,7 @@ static void extent_to_replicas(struct bkey_s_c k, continue; if (!p.has_ec) - r->devs[r->nr_devs++] = p.ptr.dev; + replicas_entry_add_dev(r, p.ptr.dev); else r->nr_required = 0; } @@ -140,7 +140,7 @@ static void stripe_to_replicas(struct bkey_s_c k, for (ptr = s.v->ptrs; ptr < s.v->ptrs + s.v->nr_blocks; ptr++) - r->devs[r->nr_devs++] = ptr->dev; + replicas_entry_add_dev(r, ptr->dev); } void bch2_bkey_to_replicas(struct bch_replicas_entry_v1 *e, @@ -181,7 +181,7 @@ void bch2_devlist_to_replicas(struct bch_replicas_entry_v1 *e, e->nr_required = 1; darray_for_each(devs, i) - e->devs[e->nr_devs++] = *i; + replicas_entry_add_dev(e, *i); bch2_replicas_entry_sort(e); } diff --git a/fs/bcachefs/replicas_format.h b/fs/bcachefs/replicas_format.h index b97208195d06..b7eff904acdb 100644 --- a/fs/bcachefs/replicas_format.h +++ b/fs/bcachefs/replicas_format.h @@ -5,7 +5,7 @@ struct bch_replicas_entry_v0 { __u8 data_type; __u8 nr_devs; - __u8 devs[]; + __u8 devs[] __counted_by(nr_devs); } __packed; struct bch_sb_field_replicas_v0 { @@ -17,7 +17,7 @@ struct bch_replicas_entry_v1 { __u8 data_type; __u8 nr_devs; __u8 nr_required; - __u8 devs[]; + __u8 devs[] __counted_by(nr_devs); } __packed; struct bch_sb_field_replicas { @@ -28,4 +28,9 @@ struct bch_sb_field_replicas { #define replicas_entry_bytes(_i) \ (offsetof(typeof(*(_i)), devs) + (_i)->nr_devs) +#define replicas_entry_add_dev(e, d) ({ \ + (e)->nr_devs++; \ + (e)->devs[(e)->nr_devs - 1] = (d); \ +}) + #endif /* _BCACHEFS_REPLICAS_FORMAT_H */ From 8037ac08c2bbb3186f83a5a924f52d1048dbaec5 Mon Sep 17 00:00:00 2001 From: "Maciej W. Rozycki" Date: Fri, 9 Aug 2024 14:24:46 +0100 Subject: [PATCH 181/302] PCI: Clear the LBMS bit after a link retrain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The LBMS bit, where implemented, is set by hardware either in response to the completion of retraining caused by writing 1 to the Retrain Link bit or whenever hardware has changed the link speed or width in attempt to correct unreliable link operation. It is never cleared by hardware other than by software writing 1 to the bit position in the Link Status register and we never do such a write. We currently have two places, namely apply_bad_link_workaround() and pcie_failed_link_retrain() in drivers/pci/controller/dwc/pcie-tegra194.c and drivers/pci/quirks.c respectively where we check the state of the LBMS bit and neither is interested in the state of the bit resulting from the completion of retraining, both check for a link fault. And in particular pcie_failed_link_retrain() causes issues consequently, by trying to retrain a link where there's no downstream device anymore and the state of 1 in the LBMS bit has been retained from when there was a device downstream that has since been removed. Clear the LBMS bit then at the conclusion of pcie_retrain_link(), so that we have a single place that controls it and that our code can track link speed or width changes resulting from unreliable link operation. Fixes: a89c82249c37 ("PCI: Work around PCIe link training failures") Link: https://lore.kernel.org/r/alpine.DEB.2.21.2408091133140.61955@angie.orcam.me.uk Reported-by: Matthew W Carlis Link: https://lore.kernel.org/r/20240806000659.30859-1-mattc@purestorage.com/ Link: https://lore.kernel.org/r/20240722193407.23255-1-mattc@purestorage.com/ Signed-off-by: Maciej W. Rozycki Signed-off-by: Bjorn Helgaas Signed-off-by: Krzysztof Wilczyński Cc: # v6.5+ --- drivers/pci/pci.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index e3a49f66982d..5618a8927aa9 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -4717,7 +4717,15 @@ int pcie_retrain_link(struct pci_dev *pdev, bool use_lt) pcie_capability_clear_word(pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_RL); } - return pcie_wait_for_link_status(pdev, use_lt, !use_lt); + rc = pcie_wait_for_link_status(pdev, use_lt, !use_lt); + + /* + * Clear LBMS after a manual retrain so that the bit can be used + * to track link speed or width changes made by hardware itself + * in attempt to correct unreliable link operation. + */ + pcie_capability_write_word(pdev, PCI_EXP_LNKSTA, PCI_EXP_LNKSTA_LBMS); + return rc; } /** From f68dea13405c94381d08f42dbf0416261622bdad Mon Sep 17 00:00:00 2001 From: "Maciej W. Rozycki" Date: Fri, 9 Aug 2024 14:24:51 +0100 Subject: [PATCH 182/302] PCI: Revert to the original speed after PCIe failed link retraining MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When `pcie_failed_link_retrain' has failed to retrain the link by hand it leaves the link speed restricted to 2.5GT/s, which will then affect any device that has been plugged in later on, which may not suffer from the problem that caused the speed restriction to have been attempted. Consequently such a downstream device will suffer from an unnecessary communication throughput limitation and therefore performance loss. Remove the speed restriction then and revert the Link Control 2 register to its original state if link retraining with the speed restriction in place has failed. Retrain the link again afterwards so as to remove any residual state, waiting on LT rather than DLLLA to avoid an excessive delay and ignoring the result as this training is supposed to fail anyway. Fixes: a89c82249c37 ("PCI: Work around PCIe link training failures") Link: https://lore.kernel.org/linux-pci/alpine.DEB.2.21.2408251412590.30766@angie.orcam.me.uk Reported-by: Matthew W Carlis Link: https://lore.kernel.org/r/20240806000659.30859-1-mattc@purestorage.com/ Link: https://lore.kernel.org/r/20240722193407.23255-1-mattc@purestorage.com/ Signed-off-by: Maciej W. Rozycki Signed-off-by: Bjorn Helgaas Signed-off-by: Krzysztof Wilczyński Reviewed-by: Ilpo Järvinen Cc: # v6.5+ --- drivers/pci/quirks.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index a2ce4e08edf5..ee86793109ff 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -66,7 +66,7 @@ * apply this erratum workaround to any downstream ports as long as they * support Link Active reporting and have the Link Control 2 register. * Restrict the speed to 2.5GT/s then with the Target Link Speed field, - * request a retrain and wait 200ms for the data link to go up. + * request a retrain and check the result. * * If this turns out successful and we know by the Vendor:Device ID it is * safe to do so, then lift the restriction, letting the devices negotiate @@ -74,6 +74,10 @@ * firmware may have already arranged and lift it with ports that already * report their data link being up. * + * Otherwise revert the speed to the original setting and request a retrain + * again to remove any residual state, ignoring the result as it's supposed + * to fail anyway. + * * Return TRUE if the link has been successfully retrained, otherwise FALSE. */ bool pcie_failed_link_retrain(struct pci_dev *dev) @@ -92,6 +96,8 @@ bool pcie_failed_link_retrain(struct pci_dev *dev) pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &lnksta); if ((lnksta & (PCI_EXP_LNKSTA_LBMS | PCI_EXP_LNKSTA_DLLLA)) == PCI_EXP_LNKSTA_LBMS) { + u16 oldlnkctl2 = lnkctl2; + pci_info(dev, "broken device, retraining non-functional downstream link at 2.5GT/s\n"); lnkctl2 &= ~PCI_EXP_LNKCTL2_TLS; @@ -100,6 +106,9 @@ bool pcie_failed_link_retrain(struct pci_dev *dev) if (pcie_retrain_link(dev, false)) { pci_info(dev, "retraining failed\n"); + pcie_capability_write_word(dev, PCI_EXP_LNKCTL2, + oldlnkctl2); + pcie_retrain_link(dev, true); return false; } From 712e49c967064a3a7a5738c6f65ac540a3f6a1df Mon Sep 17 00:00:00 2001 From: "Maciej W. Rozycki" Date: Fri, 9 Aug 2024 14:24:56 +0100 Subject: [PATCH 183/302] PCI: Correct error reporting with PCIe failed link retraining MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only return successful completion status from pcie_failed_link_retrain() if retraining has actually been done, preventing excessive delays from being triggered at call sites in a hope that communication will finally be established with the downstream device where in fact nothing has been done about the link in question that would justify such a hope. Fixes: a89c82249c37 ("PCI: Work around PCIe link training failures") Link: https://lore.kernel.org/r/alpine.DEB.2.21.2408091133260.61955@angie.orcam.me.uk Reported-by: Ilpo Järvinen Link: https://lore.kernel.org/r/aa2d1c4e-9961-d54a-00c7-ddf8e858a9b0@linux.intel.com/ Signed-off-by: Maciej W. Rozycki Signed-off-by: Bjorn Helgaas Signed-off-by: Krzysztof Wilczyński Reviewed-by: Ilpo Järvinen Cc: # v6.5+ --- drivers/pci/quirks.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index ee86793109ff..4d5483e9f63f 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -78,7 +78,8 @@ * again to remove any residual state, ignoring the result as it's supposed * to fail anyway. * - * Return TRUE if the link has been successfully retrained, otherwise FALSE. + * Return TRUE if the link has been successfully retrained. Return FALSE + * if retraining was not needed or we attempted a retrain and it failed. */ bool pcie_failed_link_retrain(struct pci_dev *dev) { @@ -87,6 +88,7 @@ bool pcie_failed_link_retrain(struct pci_dev *dev) {} }; u16 lnksta, lnkctl2; + bool ret = false; if (!pci_is_pcie(dev) || !pcie_downstream_port(dev) || !pcie_cap_has_lnkctl2(dev) || !dev->link_active_reporting) @@ -104,7 +106,8 @@ bool pcie_failed_link_retrain(struct pci_dev *dev) lnkctl2 |= PCI_EXP_LNKCTL2_TLS_2_5GT; pcie_capability_write_word(dev, PCI_EXP_LNKCTL2, lnkctl2); - if (pcie_retrain_link(dev, false)) { + ret = pcie_retrain_link(dev, false) == 0; + if (!ret) { pci_info(dev, "retraining failed\n"); pcie_capability_write_word(dev, PCI_EXP_LNKCTL2, oldlnkctl2); @@ -126,13 +129,14 @@ bool pcie_failed_link_retrain(struct pci_dev *dev) lnkctl2 |= lnkcap & PCI_EXP_LNKCAP_SLS; pcie_capability_write_word(dev, PCI_EXP_LNKCTL2, lnkctl2); - if (pcie_retrain_link(dev, false)) { + ret = pcie_retrain_link(dev, false) == 0; + if (!ret) { pci_info(dev, "retraining failed\n"); return false; } } - return true; + return ret; } static ktime_t fixup_debug_start(struct pci_dev *dev, From 59100eb248c0b15585affa546c7f6834b30eb5a4 Mon Sep 17 00:00:00 2001 From: "Maciej W. Rozycki" Date: Fri, 9 Aug 2024 14:25:02 +0100 Subject: [PATCH 184/302] PCI: Use an error code with PCIe failed link retraining MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Given how the call place in pcie_wait_for_link_delay() got structured now, and that pcie_retrain_link() returns a potentially useful error code, convert pcie_failed_link_retrain() to return an error code rather than a boolean status, fixing handling at the call site mentioned. Update the other call site accordingly. Fixes: 1abb47390350 ("Merge branch 'pci/enumeration'") Link: https://lore.kernel.org/r/alpine.DEB.2.21.2408091156530.61955@angie.orcam.me.uk Reported-by: Ilpo Järvinen Link: https://lore.kernel.org/r/aa2d1c4e-9961-d54a-00c7-ddf8e858a9b0@linux.intel.com/ Signed-off-by: Maciej W. Rozycki Signed-off-by: Bjorn Helgaas Signed-off-by: Krzysztof Wilczyński Reviewed-by: Ilpo Järvinen Cc: # v6.5+ --- drivers/pci/pci.c | 2 +- drivers/pci/pci.h | 6 +++--- drivers/pci/quirks.c | 20 ++++++++++---------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 5618a8927aa9..d46652760b11 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -1324,7 +1324,7 @@ static int pci_dev_wait(struct pci_dev *dev, char *reset_type, int timeout) if (delay > PCI_RESET_WAIT) { if (retrain) { retrain = false; - if (pcie_failed_link_retrain(bridge)) { + if (pcie_failed_link_retrain(bridge) == 0) { delay = 1; continue; } diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 79c8398f3938..7c06e55c5072 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -606,7 +606,7 @@ void pci_acs_init(struct pci_dev *dev); int pci_dev_specific_acs_enabled(struct pci_dev *dev, u16 acs_flags); int pci_dev_specific_enable_acs(struct pci_dev *dev); int pci_dev_specific_disable_acs_redir(struct pci_dev *dev); -bool pcie_failed_link_retrain(struct pci_dev *dev); +int pcie_failed_link_retrain(struct pci_dev *dev); #else static inline int pci_dev_specific_acs_enabled(struct pci_dev *dev, u16 acs_flags) @@ -621,9 +621,9 @@ static inline int pci_dev_specific_disable_acs_redir(struct pci_dev *dev) { return -ENOTTY; } -static inline bool pcie_failed_link_retrain(struct pci_dev *dev) +static inline int pcie_failed_link_retrain(struct pci_dev *dev) { - return false; + return -ENOTTY; } #endif diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index 4d5483e9f63f..5d57ea27dbc4 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -78,21 +78,21 @@ * again to remove any residual state, ignoring the result as it's supposed * to fail anyway. * - * Return TRUE if the link has been successfully retrained. Return FALSE + * Return 0 if the link has been successfully retrained. Return an error * if retraining was not needed or we attempted a retrain and it failed. */ -bool pcie_failed_link_retrain(struct pci_dev *dev) +int pcie_failed_link_retrain(struct pci_dev *dev) { static const struct pci_device_id ids[] = { { PCI_VDEVICE(ASMEDIA, 0x2824) }, /* ASMedia ASM2824 */ {} }; u16 lnksta, lnkctl2; - bool ret = false; + int ret = -ENOTTY; if (!pci_is_pcie(dev) || !pcie_downstream_port(dev) || !pcie_cap_has_lnkctl2(dev) || !dev->link_active_reporting) - return false; + return ret; pcie_capability_read_word(dev, PCI_EXP_LNKCTL2, &lnkctl2); pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &lnksta); @@ -106,13 +106,13 @@ bool pcie_failed_link_retrain(struct pci_dev *dev) lnkctl2 |= PCI_EXP_LNKCTL2_TLS_2_5GT; pcie_capability_write_word(dev, PCI_EXP_LNKCTL2, lnkctl2); - ret = pcie_retrain_link(dev, false) == 0; - if (!ret) { + ret = pcie_retrain_link(dev, false); + if (ret) { pci_info(dev, "retraining failed\n"); pcie_capability_write_word(dev, PCI_EXP_LNKCTL2, oldlnkctl2); pcie_retrain_link(dev, true); - return false; + return ret; } pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &lnksta); @@ -129,10 +129,10 @@ bool pcie_failed_link_retrain(struct pci_dev *dev) lnkctl2 |= lnkcap & PCI_EXP_LNKCAP_SLS; pcie_capability_write_word(dev, PCI_EXP_LNKCTL2, lnkctl2); - ret = pcie_retrain_link(dev, false) == 0; - if (!ret) { + ret = pcie_retrain_link(dev, false); + if (ret) { pci_info(dev, "retraining failed\n"); - return false; + return ret; } } From 5214ff221a14cadab1e2ee29499750fd5e884feb Mon Sep 17 00:00:00 2001 From: Richard Zhu Date: Mon, 29 Jul 2024 16:18:08 -0400 Subject: [PATCH 185/302] PCI: imx6: Fix establish link failure in EP mode for i.MX8MM and i.MX8MP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add IMX6_PCIE_FLAG_HAS_APP_RESET flag to IMX8MM_EP and IMX8MP_EP drvdata. This flag was overlooked during code restructuring. It is crucial to release the app-reset from the System Reset Controller before initiating LTSSM to rectify the issue. Fixes: 0c9651c21f2a ("PCI: imx6: Simplify reset handling by using *_FLAG_HAS_*_RESET") Link: https://lore.kernel.org/linux-pci/20240729-pci2_upstream-v8-1-b68ee5ef2b4d@nxp.com Signed-off-by: Richard Zhu Signed-off-by: Frank Li [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Manivannan Sadhasivam Cc: # 6.9+ --- drivers/pci/controller/dwc/pci-imx6.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c index 964d67756eb2..42fd17fbadfa 100644 --- a/drivers/pci/controller/dwc/pci-imx6.c +++ b/drivers/pci/controller/dwc/pci-imx6.c @@ -1562,7 +1562,8 @@ static const struct imx6_pcie_drvdata drvdata[] = { }, [IMX8MM_EP] = { .variant = IMX8MM_EP, - .flags = IMX6_PCIE_FLAG_HAS_PHYDRV, + .flags = IMX6_PCIE_FLAG_HAS_APP_RESET | + IMX6_PCIE_FLAG_HAS_PHYDRV, .mode = DW_PCIE_EP_TYPE, .gpr = "fsl,imx8mm-iomuxc-gpr", .clk_names = imx8mm_clks, @@ -1573,7 +1574,8 @@ static const struct imx6_pcie_drvdata drvdata[] = { }, [IMX8MP_EP] = { .variant = IMX8MP_EP, - .flags = IMX6_PCIE_FLAG_HAS_PHYDRV, + .flags = IMX6_PCIE_FLAG_HAS_APP_RESET | + IMX6_PCIE_FLAG_HAS_PHYDRV, .mode = DW_PCIE_EP_TYPE, .gpr = "fsl,imx8mp-iomuxc-gpr", .clk_names = imx8mm_clks, From 5cb3aa92c7cf182940ae575c3f450d3708af087c Mon Sep 17 00:00:00 2001 From: Richard Zhu Date: Mon, 29 Jul 2024 16:18:09 -0400 Subject: [PATCH 186/302] PCI: imx6: Fix i.MX8MP PCIe EP's occasional failure to trigger MSI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Correct occasional MSI triggering failures in i.MX8MP PCIe EP by applying the correct hardware outbound alignment requirement. The i.MX platform has a restriction about outbound address translation. The pci-epc-mem uses page_size to manage it. Set the correct page_size for i.MX platform to meet the hardware requirement, which is the same as inbound address alignment. Thus, align it with epc_features::align. Fixes: 1bd0d43dcf3b ("PCI: imx6: Clean up addr_space retrieval code") Link: https://lore.kernel.org/linux-pci/20240729-pci2_upstream-v8-2-b68ee5ef2b4d@nxp.com Signed-off-by: Richard Zhu Signed-off-by: Frank Li [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Manivannan Sadhasivam Acked-by: Jason Liu Cc: # 6.9+ --- drivers/pci/controller/dwc/pci-imx6.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c index 42fd17fbadfa..3b739aa7c516 100644 --- a/drivers/pci/controller/dwc/pci-imx6.c +++ b/drivers/pci/controller/dwc/pci-imx6.c @@ -1113,6 +1113,8 @@ static int imx6_add_pcie_ep(struct imx6_pcie *imx6_pcie, if (imx6_check_flag(imx6_pcie, IMX6_PCIE_FLAG_SUPPORT_64BIT)) dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); + ep->page_size = imx6_pcie->drvdata->epc_features->align; + ret = dw_pcie_ep_init(ep); if (ret) { dev_err(dev, "failed to initialize endpoint\n"); From 5b04d44d5c74e4d8aab1678496b84700b4b343fe Mon Sep 17 00:00:00 2001 From: Frank Li Date: Mon, 29 Jul 2024 16:18:10 -0400 Subject: [PATCH 187/302] PCI: imx6: Fix missing call to phy_power_off() in error handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix missing call to phy_power_off() in the error path of imx6_pcie_host_init(). Remove unnecessary check for imx6_pcie->phy as the PHY API already handles NULL pointers. Fixes: cbcf8722b523 ("phy: freescale: imx8m-pcie: Fix the wrong order of phy_init() and phy_power_on()") Link: https://lore.kernel.org/linux-pci/20240729-pci2_upstream-v8-3-b68ee5ef2b4d@nxp.com Signed-off-by: Frank Li [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Manivannan Sadhasivam Cc: # 6.1+ --- drivers/pci/controller/dwc/pci-imx6.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c index 3b739aa7c516..eaec471c4623 100644 --- a/drivers/pci/controller/dwc/pci-imx6.c +++ b/drivers/pci/controller/dwc/pci-imx6.c @@ -953,7 +953,7 @@ static int imx6_pcie_host_init(struct dw_pcie_rp *pp) ret = phy_power_on(imx6_pcie->phy); if (ret) { dev_err(dev, "waiting for PHY ready timeout!\n"); - goto err_phy_off; + goto err_phy_exit; } } @@ -968,8 +968,9 @@ static int imx6_pcie_host_init(struct dw_pcie_rp *pp) return 0; err_phy_off: - if (imx6_pcie->phy) - phy_exit(imx6_pcie->phy); + phy_power_off(imx6_pcie->phy); +err_phy_exit: + phy_exit(imx6_pcie->phy); err_clk_disable: imx6_pcie_clk_disable(imx6_pcie); err_reg_disable: From d657ea28d55037ef9a5e58d51aa757d981b8bfc2 Mon Sep 17 00:00:00 2001 From: Frank Li Date: Mon, 29 Jul 2024 16:18:11 -0400 Subject: [PATCH 188/302] PCI: imx6: Rename imx6_* with imx_* MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since this driver has evolved to support other i.MX SoCs such as i.MX7/8/9, thus rename the 'imx6' prefix to 'imx' to avoid confusion. The driver name is left unchanged to avoid breaking userspace. Link: https://lore.kernel.org/linux-pci/20240729-pci2_upstream-v8-3-b68ee5ef2b4d@nxp.com Signed-off-by: Frank Li [kwilczynski: commit log, refactor the IMX_* macros] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Manivannan Sadhasivam --- drivers/pci/controller/dwc/pci-imx6.c | 749 +++++++++++++------------- 1 file changed, 374 insertions(+), 375 deletions(-) diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c index eaec471c4623..2f418508ed9b 100644 --- a/drivers/pci/controller/dwc/pci-imx6.c +++ b/drivers/pci/controller/dwc/pci-imx6.c @@ -54,9 +54,9 @@ #define IMX95_PE0_GEN_CTRL_3 0x1058 #define IMX95_PCIE_LTSSM_EN BIT(0) -#define to_imx6_pcie(x) dev_get_drvdata((x)->dev) +#define to_imx_pcie(x) dev_get_drvdata((x)->dev) -enum imx6_pcie_variants { +enum imx_pcie_variants { IMX6Q, IMX6SX, IMX6QP, @@ -71,25 +71,24 @@ enum imx6_pcie_variants { IMX95_EP, }; -#define IMX6_PCIE_FLAG_IMX6_PHY BIT(0) -#define IMX6_PCIE_FLAG_IMX6_SPEED_CHANGE BIT(1) -#define IMX6_PCIE_FLAG_SUPPORTS_SUSPEND BIT(2) -#define IMX6_PCIE_FLAG_HAS_PHYDRV BIT(3) -#define IMX6_PCIE_FLAG_HAS_APP_RESET BIT(4) -#define IMX6_PCIE_FLAG_HAS_PHY_RESET BIT(5) -#define IMX6_PCIE_FLAG_HAS_SERDES BIT(6) -#define IMX6_PCIE_FLAG_SUPPORT_64BIT BIT(7) +#define IMX_PCIE_FLAG_IMX_PHY BIT(0) +#define IMX_PCIE_FLAG_IMX_SPEED_CHANGE BIT(1) +#define IMX_PCIE_FLAG_SUPPORTS_SUSPEND BIT(2) +#define IMX_PCIE_FLAG_HAS_PHYDRV BIT(3) +#define IMX_PCIE_FLAG_HAS_APP_RESET BIT(4) +#define IMX_PCIE_FLAG_HAS_PHY_RESET BIT(5) +#define IMX_PCIE_FLAG_HAS_SERDES BIT(6) +#define IMX_PCIE_FLAG_SUPPORT_64BIT BIT(7) -#define imx6_check_flag(pci, val) (pci->drvdata->flags & val) +#define imx_check_flag(pci, val) (pci->drvdata->flags & val) -#define IMX6_PCIE_MAX_CLKS 6 +#define IMX_PCIE_MAX_CLKS 6 +#define IMX_PCIE_MAX_INSTANCES 2 -#define IMX6_PCIE_MAX_INSTANCES 2 +struct imx_pcie; -struct imx6_pcie; - -struct imx6_pcie_drvdata { - enum imx6_pcie_variants variant; +struct imx_pcie_drvdata { + enum imx_pcie_variants variant; enum dw_pcie_device_mode mode; u32 flags; int dbi_length; @@ -98,17 +97,17 @@ struct imx6_pcie_drvdata { const u32 clks_cnt; const u32 ltssm_off; const u32 ltssm_mask; - const u32 mode_off[IMX6_PCIE_MAX_INSTANCES]; - const u32 mode_mask[IMX6_PCIE_MAX_INSTANCES]; + const u32 mode_off[IMX_PCIE_MAX_INSTANCES]; + const u32 mode_mask[IMX_PCIE_MAX_INSTANCES]; const struct pci_epc_features *epc_features; - int (*init_phy)(struct imx6_pcie *pcie); + int (*init_phy)(struct imx_pcie *pcie); }; -struct imx6_pcie { +struct imx_pcie { struct dw_pcie *pci; struct gpio_desc *reset_gpiod; bool link_is_up; - struct clk_bulk_data clks[IMX6_PCIE_MAX_CLKS]; + struct clk_bulk_data clks[IMX_PCIE_MAX_CLKS]; struct regmap *iomuxc_gpr; u16 msi_ctrl; u32 controller_id; @@ -129,7 +128,7 @@ struct imx6_pcie { /* power domain for pcie phy */ struct device *pd_pcie_phy; struct phy *phy; - const struct imx6_pcie_drvdata *drvdata; + const struct imx_pcie_drvdata *drvdata; }; /* Parameters for the waiting for PCIe PHY PLL to lock on i.MX7 */ @@ -184,28 +183,28 @@ struct imx6_pcie { #define PHY_RX_OVRD_IN_LO_RX_DATA_EN BIT(5) #define PHY_RX_OVRD_IN_LO_RX_PLL_EN BIT(3) -static unsigned int imx6_pcie_grp_offset(const struct imx6_pcie *imx6_pcie) +static unsigned int imx_pcie_grp_offset(const struct imx_pcie *imx_pcie) { - WARN_ON(imx6_pcie->drvdata->variant != IMX8MQ && - imx6_pcie->drvdata->variant != IMX8MQ_EP && - imx6_pcie->drvdata->variant != IMX8MM && - imx6_pcie->drvdata->variant != IMX8MM_EP && - imx6_pcie->drvdata->variant != IMX8MP && - imx6_pcie->drvdata->variant != IMX8MP_EP); - return imx6_pcie->controller_id == 1 ? IOMUXC_GPR16 : IOMUXC_GPR14; + WARN_ON(imx_pcie->drvdata->variant != IMX8MQ && + imx_pcie->drvdata->variant != IMX8MQ_EP && + imx_pcie->drvdata->variant != IMX8MM && + imx_pcie->drvdata->variant != IMX8MM_EP && + imx_pcie->drvdata->variant != IMX8MP && + imx_pcie->drvdata->variant != IMX8MP_EP); + return imx_pcie->controller_id == 1 ? IOMUXC_GPR16 : IOMUXC_GPR14; } -static int imx95_pcie_init_phy(struct imx6_pcie *imx6_pcie) +static int imx95_pcie_init_phy(struct imx_pcie *imx_pcie) { - regmap_update_bits(imx6_pcie->iomuxc_gpr, + regmap_update_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_SS_RW_REG_0, IMX95_PCIE_PHY_CR_PARA_SEL, IMX95_PCIE_PHY_CR_PARA_SEL); - regmap_update_bits(imx6_pcie->iomuxc_gpr, + regmap_update_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_PHY_GEN_CTRL, IMX95_PCIE_REF_USE_PAD, 0); - regmap_update_bits(imx6_pcie->iomuxc_gpr, + regmap_update_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_SS_RW_REG_0, IMX95_PCIE_REF_CLKEN, IMX95_PCIE_REF_CLKEN); @@ -213,9 +212,9 @@ static int imx95_pcie_init_phy(struct imx6_pcie *imx6_pcie) return 0; } -static void imx6_pcie_configure_type(struct imx6_pcie *imx6_pcie) +static void imx_pcie_configure_type(struct imx_pcie *imx_pcie) { - const struct imx6_pcie_drvdata *drvdata = imx6_pcie->drvdata; + const struct imx_pcie_drvdata *drvdata = imx_pcie->drvdata; unsigned int mask, val, mode, id; if (drvdata->mode == DW_PCIE_EP_TYPE) @@ -223,7 +222,7 @@ static void imx6_pcie_configure_type(struct imx6_pcie *imx6_pcie) else mode = PCI_EXP_TYPE_ROOT_PORT; - id = imx6_pcie->controller_id; + id = imx_pcie->controller_id; /* If mode_mask[id] is zero, means each controller have its individual gpr */ if (!drvdata->mode_mask[id]) @@ -232,12 +231,12 @@ static void imx6_pcie_configure_type(struct imx6_pcie *imx6_pcie) mask = drvdata->mode_mask[id]; val = mode << (ffs(mask) - 1); - regmap_update_bits(imx6_pcie->iomuxc_gpr, drvdata->mode_off[id], mask, val); + regmap_update_bits(imx_pcie->iomuxc_gpr, drvdata->mode_off[id], mask, val); } -static int pcie_phy_poll_ack(struct imx6_pcie *imx6_pcie, bool exp_val) +static int pcie_phy_poll_ack(struct imx_pcie *imx_pcie, bool exp_val) { - struct dw_pcie *pci = imx6_pcie->pci; + struct dw_pcie *pci = imx_pcie->pci; bool val; u32 max_iterations = 10; u32 wait_counter = 0; @@ -256,9 +255,9 @@ static int pcie_phy_poll_ack(struct imx6_pcie *imx6_pcie, bool exp_val) return -ETIMEDOUT; } -static int pcie_phy_wait_ack(struct imx6_pcie *imx6_pcie, int addr) +static int pcie_phy_wait_ack(struct imx_pcie *imx_pcie, int addr) { - struct dw_pcie *pci = imx6_pcie->pci; + struct dw_pcie *pci = imx_pcie->pci; u32 val; int ret; @@ -268,24 +267,24 @@ static int pcie_phy_wait_ack(struct imx6_pcie *imx6_pcie, int addr) val |= PCIE_PHY_CTRL_CAP_ADR; dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, val); - ret = pcie_phy_poll_ack(imx6_pcie, true); + ret = pcie_phy_poll_ack(imx_pcie, true); if (ret) return ret; val = PCIE_PHY_CTRL_DATA(addr); dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, val); - return pcie_phy_poll_ack(imx6_pcie, false); + return pcie_phy_poll_ack(imx_pcie, false); } /* Read from the 16-bit PCIe PHY control registers (not memory-mapped) */ -static int pcie_phy_read(struct imx6_pcie *imx6_pcie, int addr, u16 *data) +static int pcie_phy_read(struct imx_pcie *imx_pcie, int addr, u16 *data) { - struct dw_pcie *pci = imx6_pcie->pci; + struct dw_pcie *pci = imx_pcie->pci; u32 phy_ctl; int ret; - ret = pcie_phy_wait_ack(imx6_pcie, addr); + ret = pcie_phy_wait_ack(imx_pcie, addr); if (ret) return ret; @@ -293,7 +292,7 @@ static int pcie_phy_read(struct imx6_pcie *imx6_pcie, int addr, u16 *data) phy_ctl = PCIE_PHY_CTRL_RD; dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, phy_ctl); - ret = pcie_phy_poll_ack(imx6_pcie, true); + ret = pcie_phy_poll_ack(imx_pcie, true); if (ret) return ret; @@ -302,18 +301,18 @@ static int pcie_phy_read(struct imx6_pcie *imx6_pcie, int addr, u16 *data) /* deassert Read signal */ dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, 0x00); - return pcie_phy_poll_ack(imx6_pcie, false); + return pcie_phy_poll_ack(imx_pcie, false); } -static int pcie_phy_write(struct imx6_pcie *imx6_pcie, int addr, u16 data) +static int pcie_phy_write(struct imx_pcie *imx_pcie, int addr, u16 data) { - struct dw_pcie *pci = imx6_pcie->pci; + struct dw_pcie *pci = imx_pcie->pci; u32 var; int ret; /* write addr */ /* cap addr */ - ret = pcie_phy_wait_ack(imx6_pcie, addr); + ret = pcie_phy_wait_ack(imx_pcie, addr); if (ret) return ret; @@ -324,7 +323,7 @@ static int pcie_phy_write(struct imx6_pcie *imx6_pcie, int addr, u16 data) var |= PCIE_PHY_CTRL_CAP_DAT; dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var); - ret = pcie_phy_poll_ack(imx6_pcie, true); + ret = pcie_phy_poll_ack(imx_pcie, true); if (ret) return ret; @@ -333,7 +332,7 @@ static int pcie_phy_write(struct imx6_pcie *imx6_pcie, int addr, u16 data) dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var); /* wait for ack de-assertion */ - ret = pcie_phy_poll_ack(imx6_pcie, false); + ret = pcie_phy_poll_ack(imx_pcie, false); if (ret) return ret; @@ -342,7 +341,7 @@ static int pcie_phy_write(struct imx6_pcie *imx6_pcie, int addr, u16 data) dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var); /* wait for ack */ - ret = pcie_phy_poll_ack(imx6_pcie, true); + ret = pcie_phy_poll_ack(imx_pcie, true); if (ret) return ret; @@ -351,7 +350,7 @@ static int pcie_phy_write(struct imx6_pcie *imx6_pcie, int addr, u16 data) dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var); /* wait for ack de-assertion */ - ret = pcie_phy_poll_ack(imx6_pcie, false); + ret = pcie_phy_poll_ack(imx_pcie, false); if (ret) return ret; @@ -360,74 +359,74 @@ static int pcie_phy_write(struct imx6_pcie *imx6_pcie, int addr, u16 data) return 0; } -static int imx8mq_pcie_init_phy(struct imx6_pcie *imx6_pcie) +static int imx8mq_pcie_init_phy(struct imx_pcie *imx_pcie) { /* TODO: Currently this code assumes external oscillator is being used */ - regmap_update_bits(imx6_pcie->iomuxc_gpr, - imx6_pcie_grp_offset(imx6_pcie), + regmap_update_bits(imx_pcie->iomuxc_gpr, + imx_pcie_grp_offset(imx_pcie), IMX8MQ_GPR_PCIE_REF_USE_PAD, IMX8MQ_GPR_PCIE_REF_USE_PAD); /* * Regarding the datasheet, the PCIE_VPH is suggested to be 1.8V. If the PCIE_VPH is * supplied by 3.3V, the VREG_BYPASS should be cleared to zero. */ - if (imx6_pcie->vph && regulator_get_voltage(imx6_pcie->vph) > 3000000) - regmap_update_bits(imx6_pcie->iomuxc_gpr, - imx6_pcie_grp_offset(imx6_pcie), + if (imx_pcie->vph && regulator_get_voltage(imx_pcie->vph) > 3000000) + regmap_update_bits(imx_pcie->iomuxc_gpr, + imx_pcie_grp_offset(imx_pcie), IMX8MQ_GPR_PCIE_VREG_BYPASS, 0); return 0; } -static int imx7d_pcie_init_phy(struct imx6_pcie *imx6_pcie) +static int imx7d_pcie_init_phy(struct imx_pcie *imx_pcie) { - regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12, IMX7D_GPR12_PCIE_PHY_REFCLK_SEL, 0); + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12, IMX7D_GPR12_PCIE_PHY_REFCLK_SEL, 0); return 0; } -static int imx6_pcie_init_phy(struct imx6_pcie *imx6_pcie) +static int imx_pcie_init_phy(struct imx_pcie *imx_pcie) { - regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12, + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12, IMX6Q_GPR12_PCIE_CTL_2, 0 << 10); /* configure constant input signal to the pcie ctrl and phy */ - regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12, + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12, IMX6Q_GPR12_LOS_LEVEL, 9 << 4); - regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8, + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR8, IMX6Q_GPR8_TX_DEEMPH_GEN1, - imx6_pcie->tx_deemph_gen1 << 0); - regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8, + imx_pcie->tx_deemph_gen1 << 0); + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR8, IMX6Q_GPR8_TX_DEEMPH_GEN2_3P5DB, - imx6_pcie->tx_deemph_gen2_3p5db << 6); - regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8, + imx_pcie->tx_deemph_gen2_3p5db << 6); + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR8, IMX6Q_GPR8_TX_DEEMPH_GEN2_6DB, - imx6_pcie->tx_deemph_gen2_6db << 12); - regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8, + imx_pcie->tx_deemph_gen2_6db << 12); + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR8, IMX6Q_GPR8_TX_SWING_FULL, - imx6_pcie->tx_swing_full << 18); - regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8, + imx_pcie->tx_swing_full << 18); + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR8, IMX6Q_GPR8_TX_SWING_LOW, - imx6_pcie->tx_swing_low << 25); + imx_pcie->tx_swing_low << 25); return 0; } -static int imx6sx_pcie_init_phy(struct imx6_pcie *imx6_pcie) +static int imx6sx_pcie_init_phy(struct imx_pcie *imx_pcie) { - regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12, + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12, IMX6SX_GPR12_PCIE_RX_EQ_MASK, IMX6SX_GPR12_PCIE_RX_EQ_2); - return imx6_pcie_init_phy(imx6_pcie); + return imx_pcie_init_phy(imx_pcie); } -static void imx7d_pcie_wait_for_phy_pll_lock(struct imx6_pcie *imx6_pcie) +static void imx7d_pcie_wait_for_phy_pll_lock(struct imx_pcie *imx_pcie) { u32 val; - struct device *dev = imx6_pcie->pci->dev; + struct device *dev = imx_pcie->pci->dev; - if (regmap_read_poll_timeout(imx6_pcie->iomuxc_gpr, + if (regmap_read_poll_timeout(imx_pcie->iomuxc_gpr, IOMUXC_GPR22, val, val & IMX7D_GPR22_PCIE_PHY_PLL_LOCKED, PHY_PLL_LOCK_WAIT_USLEEP_MAX, @@ -435,19 +434,19 @@ static void imx7d_pcie_wait_for_phy_pll_lock(struct imx6_pcie *imx6_pcie) dev_err(dev, "PCIe PLL lock timeout\n"); } -static int imx6_setup_phy_mpll(struct imx6_pcie *imx6_pcie) +static int imx_setup_phy_mpll(struct imx_pcie *imx_pcie) { unsigned long phy_rate = 0; int mult, div; u16 val; int i; - if (!(imx6_pcie->drvdata->flags & IMX6_PCIE_FLAG_IMX6_PHY)) + if (!(imx_pcie->drvdata->flags & IMX_PCIE_FLAG_IMX_PHY)) return 0; - for (i = 0; i < imx6_pcie->drvdata->clks_cnt; i++) - if (strncmp(imx6_pcie->clks[i].id, "pcie_phy", 8) == 0) - phy_rate = clk_get_rate(imx6_pcie->clks[i].clk); + for (i = 0; i < imx_pcie->drvdata->clks_cnt; i++) + if (strncmp(imx_pcie->clks[i].id, "pcie_phy", 8) == 0) + phy_rate = clk_get_rate(imx_pcie->clks[i].clk); switch (phy_rate) { case 125000000: @@ -465,46 +464,46 @@ static int imx6_setup_phy_mpll(struct imx6_pcie *imx6_pcie) div = 1; break; default: - dev_err(imx6_pcie->pci->dev, + dev_err(imx_pcie->pci->dev, "Unsupported PHY reference clock rate %lu\n", phy_rate); return -EINVAL; } - pcie_phy_read(imx6_pcie, PCIE_PHY_MPLL_OVRD_IN_LO, &val); + pcie_phy_read(imx_pcie, PCIE_PHY_MPLL_OVRD_IN_LO, &val); val &= ~(PCIE_PHY_MPLL_MULTIPLIER_MASK << PCIE_PHY_MPLL_MULTIPLIER_SHIFT); val |= mult << PCIE_PHY_MPLL_MULTIPLIER_SHIFT; val |= PCIE_PHY_MPLL_MULTIPLIER_OVRD; - pcie_phy_write(imx6_pcie, PCIE_PHY_MPLL_OVRD_IN_LO, val); + pcie_phy_write(imx_pcie, PCIE_PHY_MPLL_OVRD_IN_LO, val); - pcie_phy_read(imx6_pcie, PCIE_PHY_ATEOVRD, &val); + pcie_phy_read(imx_pcie, PCIE_PHY_ATEOVRD, &val); val &= ~(PCIE_PHY_ATEOVRD_REF_CLKDIV_MASK << PCIE_PHY_ATEOVRD_REF_CLKDIV_SHIFT); val |= div << PCIE_PHY_ATEOVRD_REF_CLKDIV_SHIFT; val |= PCIE_PHY_ATEOVRD_EN; - pcie_phy_write(imx6_pcie, PCIE_PHY_ATEOVRD, val); + pcie_phy_write(imx_pcie, PCIE_PHY_ATEOVRD, val); return 0; } -static void imx6_pcie_reset_phy(struct imx6_pcie *imx6_pcie) +static void imx_pcie_reset_phy(struct imx_pcie *imx_pcie) { u16 tmp; - if (!(imx6_pcie->drvdata->flags & IMX6_PCIE_FLAG_IMX6_PHY)) + if (!(imx_pcie->drvdata->flags & IMX_PCIE_FLAG_IMX_PHY)) return; - pcie_phy_read(imx6_pcie, PHY_RX_OVRD_IN_LO, &tmp); + pcie_phy_read(imx_pcie, PHY_RX_OVRD_IN_LO, &tmp); tmp |= (PHY_RX_OVRD_IN_LO_RX_DATA_EN | PHY_RX_OVRD_IN_LO_RX_PLL_EN); - pcie_phy_write(imx6_pcie, PHY_RX_OVRD_IN_LO, tmp); + pcie_phy_write(imx_pcie, PHY_RX_OVRD_IN_LO, tmp); usleep_range(2000, 3000); - pcie_phy_read(imx6_pcie, PHY_RX_OVRD_IN_LO, &tmp); + pcie_phy_read(imx_pcie, PHY_RX_OVRD_IN_LO, &tmp); tmp &= ~(PHY_RX_OVRD_IN_LO_RX_DATA_EN | PHY_RX_OVRD_IN_LO_RX_PLL_EN); - pcie_phy_write(imx6_pcie, PHY_RX_OVRD_IN_LO, tmp); + pcie_phy_write(imx_pcie, PHY_RX_OVRD_IN_LO, tmp); } #ifdef CONFIG_ARM @@ -543,22 +542,22 @@ static int imx6q_pcie_abort_handler(unsigned long addr, } #endif -static int imx6_pcie_attach_pd(struct device *dev) +static int imx_pcie_attach_pd(struct device *dev) { - struct imx6_pcie *imx6_pcie = dev_get_drvdata(dev); + struct imx_pcie *imx_pcie = dev_get_drvdata(dev); struct device_link *link; /* Do nothing when in a single power domain */ if (dev->pm_domain) return 0; - imx6_pcie->pd_pcie = dev_pm_domain_attach_by_name(dev, "pcie"); - if (IS_ERR(imx6_pcie->pd_pcie)) - return PTR_ERR(imx6_pcie->pd_pcie); + imx_pcie->pd_pcie = dev_pm_domain_attach_by_name(dev, "pcie"); + if (IS_ERR(imx_pcie->pd_pcie)) + return PTR_ERR(imx_pcie->pd_pcie); /* Do nothing when power domain missing */ - if (!imx6_pcie->pd_pcie) + if (!imx_pcie->pd_pcie) return 0; - link = device_link_add(dev, imx6_pcie->pd_pcie, + link = device_link_add(dev, imx_pcie->pd_pcie, DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE); @@ -567,11 +566,11 @@ static int imx6_pcie_attach_pd(struct device *dev) return -EINVAL; } - imx6_pcie->pd_pcie_phy = dev_pm_domain_attach_by_name(dev, "pcie_phy"); - if (IS_ERR(imx6_pcie->pd_pcie_phy)) - return PTR_ERR(imx6_pcie->pd_pcie_phy); + imx_pcie->pd_pcie_phy = dev_pm_domain_attach_by_name(dev, "pcie_phy"); + if (IS_ERR(imx_pcie->pd_pcie_phy)) + return PTR_ERR(imx_pcie->pd_pcie_phy); - link = device_link_add(dev, imx6_pcie->pd_pcie_phy, + link = device_link_add(dev, imx_pcie->pd_pcie_phy, DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE); @@ -583,20 +582,20 @@ static int imx6_pcie_attach_pd(struct device *dev) return 0; } -static int imx6_pcie_enable_ref_clk(struct imx6_pcie *imx6_pcie) +static int imx_pcie_enable_ref_clk(struct imx_pcie *imx_pcie) { unsigned int offset; int ret = 0; - switch (imx6_pcie->drvdata->variant) { + switch (imx_pcie->drvdata->variant) { case IMX6SX: - regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12, + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12, IMX6SX_GPR12_PCIE_TEST_POWERDOWN, 0); break; case IMX6QP: case IMX6Q: /* power up core phy and enable ref clock */ - regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1, + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, IMX6Q_GPR1_PCIE_TEST_PD, 0 << 18); /* * the async reset input need ref clock to sync internally, @@ -605,7 +604,7 @@ static int imx6_pcie_enable_ref_clk(struct imx6_pcie *imx6_pcie) * add one ~10us delay here. */ usleep_range(10, 100); - regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1, + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, IMX6Q_GPR1_PCIE_REF_CLK_EN, 1 << 16); break; case IMX7D: @@ -618,15 +617,15 @@ static int imx6_pcie_enable_ref_clk(struct imx6_pcie *imx6_pcie) case IMX8MQ_EP: case IMX8MP: case IMX8MP_EP: - offset = imx6_pcie_grp_offset(imx6_pcie); + offset = imx_pcie_grp_offset(imx_pcie); /* * Set the over ride low and enabled * make sure that REF_CLK is turned on. */ - regmap_update_bits(imx6_pcie->iomuxc_gpr, offset, + regmap_update_bits(imx_pcie->iomuxc_gpr, offset, IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE, 0); - regmap_update_bits(imx6_pcie->iomuxc_gpr, offset, + regmap_update_bits(imx_pcie->iomuxc_gpr, offset, IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE_EN, IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE_EN); break; @@ -635,19 +634,19 @@ static int imx6_pcie_enable_ref_clk(struct imx6_pcie *imx6_pcie) return ret; } -static void imx6_pcie_disable_ref_clk(struct imx6_pcie *imx6_pcie) +static void imx_pcie_disable_ref_clk(struct imx_pcie *imx_pcie) { - switch (imx6_pcie->drvdata->variant) { + switch (imx_pcie->drvdata->variant) { case IMX6QP: case IMX6Q: - regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1, + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, IMX6Q_GPR1_PCIE_REF_CLK_EN, 0); - regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1, + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, IMX6Q_GPR1_PCIE_TEST_PD, IMX6Q_GPR1_PCIE_TEST_PD); break; case IMX7D: - regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12, + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12, IMX7D_GPR12_PCIE_PHY_REFCLK_SEL, IMX7D_GPR12_PCIE_PHY_REFCLK_SEL); break; @@ -656,17 +655,17 @@ static void imx6_pcie_disable_ref_clk(struct imx6_pcie *imx6_pcie) } } -static int imx6_pcie_clk_enable(struct imx6_pcie *imx6_pcie) +static int imx_pcie_clk_enable(struct imx_pcie *imx_pcie) { - struct dw_pcie *pci = imx6_pcie->pci; + struct dw_pcie *pci = imx_pcie->pci; struct device *dev = pci->dev; int ret; - ret = clk_bulk_prepare_enable(imx6_pcie->drvdata->clks_cnt, imx6_pcie->clks); + ret = clk_bulk_prepare_enable(imx_pcie->drvdata->clks_cnt, imx_pcie->clks); if (ret) return ret; - ret = imx6_pcie_enable_ref_clk(imx6_pcie); + ret = imx_pcie_enable_ref_clk(imx_pcie); if (ret) { dev_err(dev, "unable to enable pcie ref clock\n"); goto err_ref_clk; @@ -677,41 +676,41 @@ static int imx6_pcie_clk_enable(struct imx6_pcie *imx6_pcie) return 0; err_ref_clk: - clk_bulk_disable_unprepare(imx6_pcie->drvdata->clks_cnt, imx6_pcie->clks); + clk_bulk_disable_unprepare(imx_pcie->drvdata->clks_cnt, imx_pcie->clks); return ret; } -static void imx6_pcie_clk_disable(struct imx6_pcie *imx6_pcie) +static void imx_pcie_clk_disable(struct imx_pcie *imx_pcie) { - imx6_pcie_disable_ref_clk(imx6_pcie); - clk_bulk_disable_unprepare(imx6_pcie->drvdata->clks_cnt, imx6_pcie->clks); + imx_pcie_disable_ref_clk(imx_pcie); + clk_bulk_disable_unprepare(imx_pcie->drvdata->clks_cnt, imx_pcie->clks); } -static void imx6_pcie_assert_core_reset(struct imx6_pcie *imx6_pcie) +static void imx_pcie_assert_core_reset(struct imx_pcie *imx_pcie) { - reset_control_assert(imx6_pcie->pciephy_reset); - reset_control_assert(imx6_pcie->apps_reset); + reset_control_assert(imx_pcie->pciephy_reset); + reset_control_assert(imx_pcie->apps_reset); - switch (imx6_pcie->drvdata->variant) { + switch (imx_pcie->drvdata->variant) { case IMX6SX: - regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12, + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12, IMX6SX_GPR12_PCIE_TEST_POWERDOWN, IMX6SX_GPR12_PCIE_TEST_POWERDOWN); /* Force PCIe PHY reset */ - regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR5, + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR5, IMX6SX_GPR5_PCIE_BTNRST_RESET, IMX6SX_GPR5_PCIE_BTNRST_RESET); break; case IMX6QP: - regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1, + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, IMX6Q_GPR1_PCIE_SW_RST, IMX6Q_GPR1_PCIE_SW_RST); break; case IMX6Q: - regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1, + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, IMX6Q_GPR1_PCIE_TEST_PD, 1 << 18); - regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1, + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, IMX6Q_GPR1_PCIE_REF_CLK_EN, 0 << 16); break; default: @@ -719,45 +718,45 @@ static void imx6_pcie_assert_core_reset(struct imx6_pcie *imx6_pcie) } /* Some boards don't have PCIe reset GPIO. */ - gpiod_set_value_cansleep(imx6_pcie->reset_gpiod, 1); + gpiod_set_value_cansleep(imx_pcie->reset_gpiod, 1); } -static int imx6_pcie_deassert_core_reset(struct imx6_pcie *imx6_pcie) +static int imx_pcie_deassert_core_reset(struct imx_pcie *imx_pcie) { - struct dw_pcie *pci = imx6_pcie->pci; + struct dw_pcie *pci = imx_pcie->pci; struct device *dev = pci->dev; - reset_control_deassert(imx6_pcie->pciephy_reset); + reset_control_deassert(imx_pcie->pciephy_reset); - switch (imx6_pcie->drvdata->variant) { + switch (imx_pcie->drvdata->variant) { case IMX7D: /* Workaround for ERR010728, failure of PCI-e PLL VCO to * oscillate, especially when cold. This turns off "Duty-cycle * Corrector" and other mysterious undocumented things. */ - if (likely(imx6_pcie->phy_base)) { + if (likely(imx_pcie->phy_base)) { /* De-assert DCC_FB_EN */ writel(PCIE_PHY_CMN_REG4_DCC_FB_EN, - imx6_pcie->phy_base + PCIE_PHY_CMN_REG4); + imx_pcie->phy_base + PCIE_PHY_CMN_REG4); /* Assert RX_EQS and RX_EQS_SEL */ writel(PCIE_PHY_CMN_REG24_RX_EQ_SEL | PCIE_PHY_CMN_REG24_RX_EQ, - imx6_pcie->phy_base + PCIE_PHY_CMN_REG24); + imx_pcie->phy_base + PCIE_PHY_CMN_REG24); /* Assert ATT_MODE */ writel(PCIE_PHY_CMN_REG26_ATT_MODE, - imx6_pcie->phy_base + PCIE_PHY_CMN_REG26); + imx_pcie->phy_base + PCIE_PHY_CMN_REG26); } else { dev_warn(dev, "Unable to apply ERR010728 workaround. DT missing fsl,imx7d-pcie-phy phandle ?\n"); } - imx7d_pcie_wait_for_phy_pll_lock(imx6_pcie); + imx7d_pcie_wait_for_phy_pll_lock(imx_pcie); break; case IMX6SX: - regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR5, + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR5, IMX6SX_GPR5_PCIE_BTNRST_RESET, 0); break; case IMX6QP: - regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1, + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, IMX6Q_GPR1_PCIE_SW_RST, 0); usleep_range(200, 500); @@ -767,9 +766,9 @@ static int imx6_pcie_deassert_core_reset(struct imx6_pcie *imx6_pcie) } /* Some boards don't have PCIe reset GPIO. */ - if (imx6_pcie->reset_gpiod) { + if (imx_pcie->reset_gpiod) { msleep(100); - gpiod_set_value_cansleep(imx6_pcie->reset_gpiod, 0); + gpiod_set_value_cansleep(imx_pcie->reset_gpiod, 0); /* Wait for 100ms after PERST# deassertion (PCIe r5.0, 6.6.1) */ msleep(100); } @@ -777,9 +776,9 @@ static int imx6_pcie_deassert_core_reset(struct imx6_pcie *imx6_pcie) return 0; } -static int imx6_pcie_wait_for_speed_change(struct imx6_pcie *imx6_pcie) +static int imx_pcie_wait_for_speed_change(struct imx_pcie *imx_pcie) { - struct dw_pcie *pci = imx6_pcie->pci; + struct dw_pcie *pci = imx_pcie->pci; struct device *dev = pci->dev; u32 tmp; unsigned int retries; @@ -796,33 +795,33 @@ static int imx6_pcie_wait_for_speed_change(struct imx6_pcie *imx6_pcie) return -ETIMEDOUT; } -static void imx6_pcie_ltssm_enable(struct device *dev) +static void imx_pcie_ltssm_enable(struct device *dev) { - struct imx6_pcie *imx6_pcie = dev_get_drvdata(dev); - const struct imx6_pcie_drvdata *drvdata = imx6_pcie->drvdata; + struct imx_pcie *imx_pcie = dev_get_drvdata(dev); + const struct imx_pcie_drvdata *drvdata = imx_pcie->drvdata; if (drvdata->ltssm_mask) - regmap_update_bits(imx6_pcie->iomuxc_gpr, drvdata->ltssm_off, drvdata->ltssm_mask, + regmap_update_bits(imx_pcie->iomuxc_gpr, drvdata->ltssm_off, drvdata->ltssm_mask, drvdata->ltssm_mask); - reset_control_deassert(imx6_pcie->apps_reset); + reset_control_deassert(imx_pcie->apps_reset); } -static void imx6_pcie_ltssm_disable(struct device *dev) +static void imx_pcie_ltssm_disable(struct device *dev) { - struct imx6_pcie *imx6_pcie = dev_get_drvdata(dev); - const struct imx6_pcie_drvdata *drvdata = imx6_pcie->drvdata; + struct imx_pcie *imx_pcie = dev_get_drvdata(dev); + const struct imx_pcie_drvdata *drvdata = imx_pcie->drvdata; if (drvdata->ltssm_mask) - regmap_update_bits(imx6_pcie->iomuxc_gpr, drvdata->ltssm_off, + regmap_update_bits(imx_pcie->iomuxc_gpr, drvdata->ltssm_off, drvdata->ltssm_mask, 0); - reset_control_assert(imx6_pcie->apps_reset); + reset_control_assert(imx_pcie->apps_reset); } -static int imx6_pcie_start_link(struct dw_pcie *pci) +static int imx_pcie_start_link(struct dw_pcie *pci) { - struct imx6_pcie *imx6_pcie = to_imx6_pcie(pci); + struct imx_pcie *imx_pcie = to_imx_pcie(pci); struct device *dev = pci->dev; u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); u32 tmp; @@ -841,7 +840,7 @@ static int imx6_pcie_start_link(struct dw_pcie *pci) dw_pcie_dbi_ro_wr_dis(pci); /* Start LTSSM. */ - imx6_pcie_ltssm_enable(dev); + imx_pcie_ltssm_enable(dev); ret = dw_pcie_wait_for_link(pci); if (ret) @@ -864,8 +863,8 @@ static int imx6_pcie_start_link(struct dw_pcie *pci) dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, tmp); dw_pcie_dbi_ro_wr_dis(pci); - if (imx6_pcie->drvdata->flags & - IMX6_PCIE_FLAG_IMX6_SPEED_CHANGE) { + if (imx_pcie->drvdata->flags & + IMX_PCIE_FLAG_IMX_SPEED_CHANGE) { /* * On i.MX7, DIRECT_SPEED_CHANGE behaves differently * from i.MX6 family when no link speed transition @@ -875,7 +874,7 @@ static int imx6_pcie_start_link(struct dw_pcie *pci) * failure. */ - ret = imx6_pcie_wait_for_speed_change(imx6_pcie); + ret = imx_pcie_wait_for_speed_change(imx_pcie); if (ret) { dev_err(dev, "Failed to bring link up!\n"); goto err_reset_phy; @@ -890,37 +889,37 @@ static int imx6_pcie_start_link(struct dw_pcie *pci) dev_info(dev, "Link: Only Gen1 is enabled\n"); } - imx6_pcie->link_is_up = true; + imx_pcie->link_is_up = true; tmp = dw_pcie_readw_dbi(pci, offset + PCI_EXP_LNKSTA); dev_info(dev, "Link up, Gen%i\n", tmp & PCI_EXP_LNKSTA_CLS); return 0; err_reset_phy: - imx6_pcie->link_is_up = false; + imx_pcie->link_is_up = false; dev_dbg(dev, "PHY DEBUG_R0=0x%08x DEBUG_R1=0x%08x\n", dw_pcie_readl_dbi(pci, PCIE_PORT_DEBUG0), dw_pcie_readl_dbi(pci, PCIE_PORT_DEBUG1)); - imx6_pcie_reset_phy(imx6_pcie); + imx_pcie_reset_phy(imx_pcie); return 0; } -static void imx6_pcie_stop_link(struct dw_pcie *pci) +static void imx_pcie_stop_link(struct dw_pcie *pci) { struct device *dev = pci->dev; /* Turn off PCIe LTSSM */ - imx6_pcie_ltssm_disable(dev); + imx_pcie_ltssm_disable(dev); } -static int imx6_pcie_host_init(struct dw_pcie_rp *pp) +static int imx_pcie_host_init(struct dw_pcie_rp *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct device *dev = pci->dev; - struct imx6_pcie *imx6_pcie = to_imx6_pcie(pci); + struct imx_pcie *imx_pcie = to_imx_pcie(pci); int ret; - if (imx6_pcie->vpcie) { - ret = regulator_enable(imx6_pcie->vpcie); + if (imx_pcie->vpcie) { + ret = regulator_enable(imx_pcie->vpcie); if (ret) { dev_err(dev, "failed to enable vpcie regulator: %d\n", ret); @@ -928,84 +927,84 @@ static int imx6_pcie_host_init(struct dw_pcie_rp *pp) } } - imx6_pcie_assert_core_reset(imx6_pcie); + imx_pcie_assert_core_reset(imx_pcie); - if (imx6_pcie->drvdata->init_phy) - imx6_pcie->drvdata->init_phy(imx6_pcie); + if (imx_pcie->drvdata->init_phy) + imx_pcie->drvdata->init_phy(imx_pcie); - imx6_pcie_configure_type(imx6_pcie); + imx_pcie_configure_type(imx_pcie); - ret = imx6_pcie_clk_enable(imx6_pcie); + ret = imx_pcie_clk_enable(imx_pcie); if (ret) { dev_err(dev, "unable to enable pcie clocks: %d\n", ret); goto err_reg_disable; } - if (imx6_pcie->phy) { - ret = phy_init(imx6_pcie->phy); + if (imx_pcie->phy) { + ret = phy_init(imx_pcie->phy); if (ret) { dev_err(dev, "pcie PHY power up failed\n"); goto err_clk_disable; } } - if (imx6_pcie->phy) { - ret = phy_power_on(imx6_pcie->phy); + if (imx_pcie->phy) { + ret = phy_power_on(imx_pcie->phy); if (ret) { dev_err(dev, "waiting for PHY ready timeout!\n"); goto err_phy_exit; } } - ret = imx6_pcie_deassert_core_reset(imx6_pcie); + ret = imx_pcie_deassert_core_reset(imx_pcie); if (ret < 0) { dev_err(dev, "pcie deassert core reset failed: %d\n", ret); goto err_phy_off; } - imx6_setup_phy_mpll(imx6_pcie); + imx_setup_phy_mpll(imx_pcie); return 0; err_phy_off: - phy_power_off(imx6_pcie->phy); + phy_power_off(imx_pcie->phy); err_phy_exit: - phy_exit(imx6_pcie->phy); + phy_exit(imx_pcie->phy); err_clk_disable: - imx6_pcie_clk_disable(imx6_pcie); + imx_pcie_clk_disable(imx_pcie); err_reg_disable: - if (imx6_pcie->vpcie) - regulator_disable(imx6_pcie->vpcie); + if (imx_pcie->vpcie) + regulator_disable(imx_pcie->vpcie); return ret; } -static void imx6_pcie_host_exit(struct dw_pcie_rp *pp) +static void imx_pcie_host_exit(struct dw_pcie_rp *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); - struct imx6_pcie *imx6_pcie = to_imx6_pcie(pci); + struct imx_pcie *imx_pcie = to_imx_pcie(pci); - if (imx6_pcie->phy) { - if (phy_power_off(imx6_pcie->phy)) + if (imx_pcie->phy) { + if (phy_power_off(imx_pcie->phy)) dev_err(pci->dev, "unable to power off PHY\n"); - phy_exit(imx6_pcie->phy); + phy_exit(imx_pcie->phy); } - imx6_pcie_clk_disable(imx6_pcie); + imx_pcie_clk_disable(imx_pcie); - if (imx6_pcie->vpcie) - regulator_disable(imx6_pcie->vpcie); + if (imx_pcie->vpcie) + regulator_disable(imx_pcie->vpcie); } -static const struct dw_pcie_host_ops imx6_pcie_host_ops = { - .init = imx6_pcie_host_init, - .deinit = imx6_pcie_host_exit, +static const struct dw_pcie_host_ops imx_pcie_host_ops = { + .init = imx_pcie_host_init, + .deinit = imx_pcie_host_exit, }; static const struct dw_pcie_ops dw_pcie_ops = { - .start_link = imx6_pcie_start_link, - .stop_link = imx6_pcie_stop_link, + .start_link = imx_pcie_start_link, + .stop_link = imx_pcie_stop_link, }; -static void imx6_pcie_ep_init(struct dw_pcie_ep *ep) +static void imx_pcie_ep_init(struct dw_pcie_ep *ep) { enum pci_barno bar; struct dw_pcie *pci = to_dw_pcie_from_ep(ep); @@ -1014,7 +1013,7 @@ static void imx6_pcie_ep_init(struct dw_pcie_ep *ep) dw_pcie_ep_reset_bar(pci, bar); } -static int imx6_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, +static int imx_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, unsigned int type, u16 interrupt_num) { struct dw_pcie *pci = to_dw_pcie_from_ep(ep); @@ -1061,35 +1060,35 @@ static const struct pci_epc_features imx95_pcie_epc_features = { }; static const struct pci_epc_features* -imx6_pcie_ep_get_features(struct dw_pcie_ep *ep) +imx_pcie_ep_get_features(struct dw_pcie_ep *ep) { struct dw_pcie *pci = to_dw_pcie_from_ep(ep); - struct imx6_pcie *imx6_pcie = to_imx6_pcie(pci); + struct imx_pcie *imx_pcie = to_imx_pcie(pci); - return imx6_pcie->drvdata->epc_features; + return imx_pcie->drvdata->epc_features; } static const struct dw_pcie_ep_ops pcie_ep_ops = { - .init = imx6_pcie_ep_init, - .raise_irq = imx6_pcie_ep_raise_irq, - .get_features = imx6_pcie_ep_get_features, + .init = imx_pcie_ep_init, + .raise_irq = imx_pcie_ep_raise_irq, + .get_features = imx_pcie_ep_get_features, }; -static int imx6_add_pcie_ep(struct imx6_pcie *imx6_pcie, +static int imx_add_pcie_ep(struct imx_pcie *imx_pcie, struct platform_device *pdev) { int ret; unsigned int pcie_dbi2_offset; struct dw_pcie_ep *ep; - struct dw_pcie *pci = imx6_pcie->pci; + struct dw_pcie *pci = imx_pcie->pci; struct dw_pcie_rp *pp = &pci->pp; struct device *dev = pci->dev; - imx6_pcie_host_init(pp); + imx_pcie_host_init(pp); ep = &pci->ep; ep->ops = &pcie_ep_ops; - switch (imx6_pcie->drvdata->variant) { + switch (imx_pcie->drvdata->variant) { case IMX8MQ_EP: case IMX8MM_EP: case IMX8MP_EP: @@ -1111,10 +1110,10 @@ static int imx6_add_pcie_ep(struct imx6_pcie *imx6_pcie, if (device_property_match_string(dev, "reg-names", "dbi2") >= 0) pci->dbi_base2 = NULL; - if (imx6_check_flag(imx6_pcie, IMX6_PCIE_FLAG_SUPPORT_64BIT)) + if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_SUPPORT_64BIT)) dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); - ep->page_size = imx6_pcie->drvdata->epc_features->align; + ep->page_size = imx_pcie->drvdata->epc_features->align; ret = dw_pcie_ep_init(ep); if (ret) { @@ -1132,30 +1131,30 @@ static int imx6_add_pcie_ep(struct imx6_pcie *imx6_pcie, pci_epc_init_notify(ep->epc); /* Start LTSSM. */ - imx6_pcie_ltssm_enable(dev); + imx_pcie_ltssm_enable(dev); return 0; } -static void imx6_pcie_pm_turnoff(struct imx6_pcie *imx6_pcie) +static void imx_pcie_pm_turnoff(struct imx_pcie *imx_pcie) { - struct device *dev = imx6_pcie->pci->dev; + struct device *dev = imx_pcie->pci->dev; /* Some variants have a turnoff reset in DT */ - if (imx6_pcie->turnoff_reset) { - reset_control_assert(imx6_pcie->turnoff_reset); - reset_control_deassert(imx6_pcie->turnoff_reset); + if (imx_pcie->turnoff_reset) { + reset_control_assert(imx_pcie->turnoff_reset); + reset_control_deassert(imx_pcie->turnoff_reset); goto pm_turnoff_sleep; } /* Others poke directly at IOMUXC registers */ - switch (imx6_pcie->drvdata->variant) { + switch (imx_pcie->drvdata->variant) { case IMX6SX: case IMX6QP: - regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12, + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12, IMX6SX_GPR12_PCIE_PM_TURN_OFF, IMX6SX_GPR12_PCIE_PM_TURN_OFF); - regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12, + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12, IMX6SX_GPR12_PCIE_PM_TURN_OFF, 0); break; default: @@ -1174,73 +1173,73 @@ pm_turnoff_sleep: usleep_range(1000, 10000); } -static void imx6_pcie_msi_save_restore(struct imx6_pcie *imx6_pcie, bool save) +static void imx_pcie_msi_save_restore(struct imx_pcie *imx_pcie, bool save) { u8 offset; u16 val; - struct dw_pcie *pci = imx6_pcie->pci; + struct dw_pcie *pci = imx_pcie->pci; if (pci_msi_enabled()) { offset = dw_pcie_find_capability(pci, PCI_CAP_ID_MSI); if (save) { val = dw_pcie_readw_dbi(pci, offset + PCI_MSI_FLAGS); - imx6_pcie->msi_ctrl = val; + imx_pcie->msi_ctrl = val; } else { dw_pcie_dbi_ro_wr_en(pci); - val = imx6_pcie->msi_ctrl; + val = imx_pcie->msi_ctrl; dw_pcie_writew_dbi(pci, offset + PCI_MSI_FLAGS, val); dw_pcie_dbi_ro_wr_dis(pci); } } } -static int imx6_pcie_suspend_noirq(struct device *dev) +static int imx_pcie_suspend_noirq(struct device *dev) { - struct imx6_pcie *imx6_pcie = dev_get_drvdata(dev); - struct dw_pcie_rp *pp = &imx6_pcie->pci->pp; + struct imx_pcie *imx_pcie = dev_get_drvdata(dev); + struct dw_pcie_rp *pp = &imx_pcie->pci->pp; - if (!(imx6_pcie->drvdata->flags & IMX6_PCIE_FLAG_SUPPORTS_SUSPEND)) + if (!(imx_pcie->drvdata->flags & IMX_PCIE_FLAG_SUPPORTS_SUSPEND)) return 0; - imx6_pcie_msi_save_restore(imx6_pcie, true); - imx6_pcie_pm_turnoff(imx6_pcie); - imx6_pcie_stop_link(imx6_pcie->pci); - imx6_pcie_host_exit(pp); + imx_pcie_msi_save_restore(imx_pcie, true); + imx_pcie_pm_turnoff(imx_pcie); + imx_pcie_stop_link(imx_pcie->pci); + imx_pcie_host_exit(pp); return 0; } -static int imx6_pcie_resume_noirq(struct device *dev) +static int imx_pcie_resume_noirq(struct device *dev) { int ret; - struct imx6_pcie *imx6_pcie = dev_get_drvdata(dev); - struct dw_pcie_rp *pp = &imx6_pcie->pci->pp; + struct imx_pcie *imx_pcie = dev_get_drvdata(dev); + struct dw_pcie_rp *pp = &imx_pcie->pci->pp; - if (!(imx6_pcie->drvdata->flags & IMX6_PCIE_FLAG_SUPPORTS_SUSPEND)) + if (!(imx_pcie->drvdata->flags & IMX_PCIE_FLAG_SUPPORTS_SUSPEND)) return 0; - ret = imx6_pcie_host_init(pp); + ret = imx_pcie_host_init(pp); if (ret) return ret; - imx6_pcie_msi_save_restore(imx6_pcie, false); + imx_pcie_msi_save_restore(imx_pcie, false); dw_pcie_setup_rc(pp); - if (imx6_pcie->link_is_up) - imx6_pcie_start_link(imx6_pcie->pci); + if (imx_pcie->link_is_up) + imx_pcie_start_link(imx_pcie->pci); return 0; } -static const struct dev_pm_ops imx6_pcie_pm_ops = { - NOIRQ_SYSTEM_SLEEP_PM_OPS(imx6_pcie_suspend_noirq, - imx6_pcie_resume_noirq) +static const struct dev_pm_ops imx_pcie_pm_ops = { + NOIRQ_SYSTEM_SLEEP_PM_OPS(imx_pcie_suspend_noirq, + imx_pcie_resume_noirq) }; -static int imx6_pcie_probe(struct platform_device *pdev) +static int imx_pcie_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct dw_pcie *pci; - struct imx6_pcie *imx6_pcie; + struct imx_pcie *imx_pcie; struct device_node *np; struct resource *dbi_base; struct device_node *node = dev->of_node; @@ -1248,8 +1247,8 @@ static int imx6_pcie_probe(struct platform_device *pdev) u16 val; int i; - imx6_pcie = devm_kzalloc(dev, sizeof(*imx6_pcie), GFP_KERNEL); - if (!imx6_pcie) + imx_pcie = devm_kzalloc(dev, sizeof(*imx_pcie), GFP_KERNEL); + if (!imx_pcie) return -ENOMEM; pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL); @@ -1258,10 +1257,10 @@ static int imx6_pcie_probe(struct platform_device *pdev) pci->dev = dev; pci->ops = &dw_pcie_ops; - pci->pp.ops = &imx6_pcie_host_ops; + pci->pp.ops = &imx_pcie_host_ops; - imx6_pcie->pci = pci; - imx6_pcie->drvdata = of_device_get_match_data(dev); + imx_pcie->pci = pci; + imx_pcie->drvdata = of_device_get_match_data(dev); /* Find the PHY if one is defined, only imx7d uses it */ np = of_parse_phandle(node, "fsl,imx7d-pcie-phy", 0); @@ -1273,9 +1272,9 @@ static int imx6_pcie_probe(struct platform_device *pdev) dev_err(dev, "Unable to map PCIe PHY\n"); return ret; } - imx6_pcie->phy_base = devm_ioremap_resource(dev, &res); - if (IS_ERR(imx6_pcie->phy_base)) - return PTR_ERR(imx6_pcie->phy_base); + imx_pcie->phy_base = devm_ioremap_resource(dev, &res); + if (IS_ERR(imx_pcie->phy_base)) + return PTR_ERR(imx_pcie->phy_base); } pci->dbi_base = devm_platform_get_and_ioremap_resource(pdev, 0, &dbi_base); @@ -1283,72 +1282,72 @@ static int imx6_pcie_probe(struct platform_device *pdev) return PTR_ERR(pci->dbi_base); /* Fetch GPIOs */ - imx6_pcie->reset_gpiod = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH); - if (IS_ERR(imx6_pcie->reset_gpiod)) - return dev_err_probe(dev, PTR_ERR(imx6_pcie->reset_gpiod), + imx_pcie->reset_gpiod = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH); + if (IS_ERR(imx_pcie->reset_gpiod)) + return dev_err_probe(dev, PTR_ERR(imx_pcie->reset_gpiod), "unable to get reset gpio\n"); - gpiod_set_consumer_name(imx6_pcie->reset_gpiod, "PCIe reset"); + gpiod_set_consumer_name(imx_pcie->reset_gpiod, "PCIe reset"); - if (imx6_pcie->drvdata->clks_cnt >= IMX6_PCIE_MAX_CLKS) + if (imx_pcie->drvdata->clks_cnt >= IMX_PCIE_MAX_CLKS) return dev_err_probe(dev, -ENOMEM, "clks_cnt is too big\n"); - for (i = 0; i < imx6_pcie->drvdata->clks_cnt; i++) - imx6_pcie->clks[i].id = imx6_pcie->drvdata->clk_names[i]; + for (i = 0; i < imx_pcie->drvdata->clks_cnt; i++) + imx_pcie->clks[i].id = imx_pcie->drvdata->clk_names[i]; /* Fetch clocks */ - ret = devm_clk_bulk_get(dev, imx6_pcie->drvdata->clks_cnt, imx6_pcie->clks); + ret = devm_clk_bulk_get(dev, imx_pcie->drvdata->clks_cnt, imx_pcie->clks); if (ret) return ret; - if (imx6_check_flag(imx6_pcie, IMX6_PCIE_FLAG_HAS_PHYDRV)) { - imx6_pcie->phy = devm_phy_get(dev, "pcie-phy"); - if (IS_ERR(imx6_pcie->phy)) - return dev_err_probe(dev, PTR_ERR(imx6_pcie->phy), + if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_HAS_PHYDRV)) { + imx_pcie->phy = devm_phy_get(dev, "pcie-phy"); + if (IS_ERR(imx_pcie->phy)) + return dev_err_probe(dev, PTR_ERR(imx_pcie->phy), "failed to get pcie phy\n"); } - if (imx6_check_flag(imx6_pcie, IMX6_PCIE_FLAG_HAS_APP_RESET)) { - imx6_pcie->apps_reset = devm_reset_control_get_exclusive(dev, "apps"); - if (IS_ERR(imx6_pcie->apps_reset)) - return dev_err_probe(dev, PTR_ERR(imx6_pcie->apps_reset), + if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_HAS_APP_RESET)) { + imx_pcie->apps_reset = devm_reset_control_get_exclusive(dev, "apps"); + if (IS_ERR(imx_pcie->apps_reset)) + return dev_err_probe(dev, PTR_ERR(imx_pcie->apps_reset), "failed to get pcie apps reset control\n"); } - if (imx6_check_flag(imx6_pcie, IMX6_PCIE_FLAG_HAS_PHY_RESET)) { - imx6_pcie->pciephy_reset = devm_reset_control_get_exclusive(dev, "pciephy"); - if (IS_ERR(imx6_pcie->pciephy_reset)) - return dev_err_probe(dev, PTR_ERR(imx6_pcie->pciephy_reset), + if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_HAS_PHY_RESET)) { + imx_pcie->pciephy_reset = devm_reset_control_get_exclusive(dev, "pciephy"); + if (IS_ERR(imx_pcie->pciephy_reset)) + return dev_err_probe(dev, PTR_ERR(imx_pcie->pciephy_reset), "Failed to get PCIEPHY reset control\n"); } - switch (imx6_pcie->drvdata->variant) { + switch (imx_pcie->drvdata->variant) { case IMX8MQ: case IMX8MQ_EP: case IMX7D: if (dbi_base->start == IMX8MQ_PCIE2_BASE_ADDR) - imx6_pcie->controller_id = 1; + imx_pcie->controller_id = 1; break; default: break; } /* Grab turnoff reset */ - imx6_pcie->turnoff_reset = devm_reset_control_get_optional_exclusive(dev, "turnoff"); - if (IS_ERR(imx6_pcie->turnoff_reset)) { + imx_pcie->turnoff_reset = devm_reset_control_get_optional_exclusive(dev, "turnoff"); + if (IS_ERR(imx_pcie->turnoff_reset)) { dev_err(dev, "Failed to get TURNOFF reset control\n"); - return PTR_ERR(imx6_pcie->turnoff_reset); + return PTR_ERR(imx_pcie->turnoff_reset); } - if (imx6_pcie->drvdata->gpr) { + if (imx_pcie->drvdata->gpr) { /* Grab GPR config register range */ - imx6_pcie->iomuxc_gpr = - syscon_regmap_lookup_by_compatible(imx6_pcie->drvdata->gpr); - if (IS_ERR(imx6_pcie->iomuxc_gpr)) - return dev_err_probe(dev, PTR_ERR(imx6_pcie->iomuxc_gpr), + imx_pcie->iomuxc_gpr = + syscon_regmap_lookup_by_compatible(imx_pcie->drvdata->gpr); + if (IS_ERR(imx_pcie->iomuxc_gpr)) + return dev_err_probe(dev, PTR_ERR(imx_pcie->iomuxc_gpr), "unable to find iomuxc registers\n"); } - if (imx6_check_flag(imx6_pcie, IMX6_PCIE_FLAG_HAS_SERDES)) { + if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_HAS_SERDES)) { void __iomem *off = devm_platform_ioremap_resource_byname(pdev, "app"); if (IS_ERR(off)) @@ -1361,59 +1360,59 @@ static int imx6_pcie_probe(struct platform_device *pdev) .reg_stride = 4, }; - imx6_pcie->iomuxc_gpr = devm_regmap_init_mmio(dev, off, ®map_config); - if (IS_ERR(imx6_pcie->iomuxc_gpr)) - return dev_err_probe(dev, PTR_ERR(imx6_pcie->iomuxc_gpr), + imx_pcie->iomuxc_gpr = devm_regmap_init_mmio(dev, off, ®map_config); + if (IS_ERR(imx_pcie->iomuxc_gpr)) + return dev_err_probe(dev, PTR_ERR(imx_pcie->iomuxc_gpr), "unable to find iomuxc registers\n"); } /* Grab PCIe PHY Tx Settings */ if (of_property_read_u32(node, "fsl,tx-deemph-gen1", - &imx6_pcie->tx_deemph_gen1)) - imx6_pcie->tx_deemph_gen1 = 0; + &imx_pcie->tx_deemph_gen1)) + imx_pcie->tx_deemph_gen1 = 0; if (of_property_read_u32(node, "fsl,tx-deemph-gen2-3p5db", - &imx6_pcie->tx_deemph_gen2_3p5db)) - imx6_pcie->tx_deemph_gen2_3p5db = 0; + &imx_pcie->tx_deemph_gen2_3p5db)) + imx_pcie->tx_deemph_gen2_3p5db = 0; if (of_property_read_u32(node, "fsl,tx-deemph-gen2-6db", - &imx6_pcie->tx_deemph_gen2_6db)) - imx6_pcie->tx_deemph_gen2_6db = 20; + &imx_pcie->tx_deemph_gen2_6db)) + imx_pcie->tx_deemph_gen2_6db = 20; if (of_property_read_u32(node, "fsl,tx-swing-full", - &imx6_pcie->tx_swing_full)) - imx6_pcie->tx_swing_full = 127; + &imx_pcie->tx_swing_full)) + imx_pcie->tx_swing_full = 127; if (of_property_read_u32(node, "fsl,tx-swing-low", - &imx6_pcie->tx_swing_low)) - imx6_pcie->tx_swing_low = 127; + &imx_pcie->tx_swing_low)) + imx_pcie->tx_swing_low = 127; /* Limit link speed */ pci->link_gen = 1; of_property_read_u32(node, "fsl,max-link-speed", &pci->link_gen); - imx6_pcie->vpcie = devm_regulator_get_optional(&pdev->dev, "vpcie"); - if (IS_ERR(imx6_pcie->vpcie)) { - if (PTR_ERR(imx6_pcie->vpcie) != -ENODEV) - return PTR_ERR(imx6_pcie->vpcie); - imx6_pcie->vpcie = NULL; + imx_pcie->vpcie = devm_regulator_get_optional(&pdev->dev, "vpcie"); + if (IS_ERR(imx_pcie->vpcie)) { + if (PTR_ERR(imx_pcie->vpcie) != -ENODEV) + return PTR_ERR(imx_pcie->vpcie); + imx_pcie->vpcie = NULL; } - imx6_pcie->vph = devm_regulator_get_optional(&pdev->dev, "vph"); - if (IS_ERR(imx6_pcie->vph)) { - if (PTR_ERR(imx6_pcie->vph) != -ENODEV) - return PTR_ERR(imx6_pcie->vph); - imx6_pcie->vph = NULL; + imx_pcie->vph = devm_regulator_get_optional(&pdev->dev, "vph"); + if (IS_ERR(imx_pcie->vph)) { + if (PTR_ERR(imx_pcie->vph) != -ENODEV) + return PTR_ERR(imx_pcie->vph); + imx_pcie->vph = NULL; } - platform_set_drvdata(pdev, imx6_pcie); + platform_set_drvdata(pdev, imx_pcie); - ret = imx6_pcie_attach_pd(dev); + ret = imx_pcie_attach_pd(dev); if (ret) return ret; - if (imx6_pcie->drvdata->mode == DW_PCIE_EP_TYPE) { - ret = imx6_add_pcie_ep(imx6_pcie, pdev); + if (imx_pcie->drvdata->mode == DW_PCIE_EP_TYPE) { + ret = imx_add_pcie_ep(imx_pcie, pdev); if (ret < 0) return ret; } else { @@ -1433,12 +1432,12 @@ static int imx6_pcie_probe(struct platform_device *pdev) return 0; } -static void imx6_pcie_shutdown(struct platform_device *pdev) +static void imx_pcie_shutdown(struct platform_device *pdev) { - struct imx6_pcie *imx6_pcie = platform_get_drvdata(pdev); + struct imx_pcie *imx_pcie = platform_get_drvdata(pdev); /* bring down link, so bootloader gets clean state in case of reboot */ - imx6_pcie_assert_core_reset(imx6_pcie); + imx_pcie_assert_core_reset(imx_pcie); } static const char * const imx6q_clks[] = {"pcie_bus", "pcie", "pcie_phy"}; @@ -1446,11 +1445,11 @@ static const char * const imx8mm_clks[] = {"pcie_bus", "pcie", "pcie_aux"}; static const char * const imx8mq_clks[] = {"pcie_bus", "pcie", "pcie_phy", "pcie_aux"}; static const char * const imx6sx_clks[] = {"pcie_bus", "pcie", "pcie_phy", "pcie_inbound_axi"}; -static const struct imx6_pcie_drvdata drvdata[] = { +static const struct imx_pcie_drvdata drvdata[] = { [IMX6Q] = { .variant = IMX6Q, - .flags = IMX6_PCIE_FLAG_IMX6_PHY | - IMX6_PCIE_FLAG_IMX6_SPEED_CHANGE, + .flags = IMX_PCIE_FLAG_IMX_PHY | + IMX_PCIE_FLAG_IMX_SPEED_CHANGE, .dbi_length = 0x200, .gpr = "fsl,imx6q-iomuxc-gpr", .clk_names = imx6q_clks, @@ -1459,13 +1458,13 @@ static const struct imx6_pcie_drvdata drvdata[] = { .ltssm_mask = IMX6Q_GPR12_PCIE_CTL_2, .mode_off[0] = IOMUXC_GPR12, .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE, - .init_phy = imx6_pcie_init_phy, + .init_phy = imx_pcie_init_phy, }, [IMX6SX] = { .variant = IMX6SX, - .flags = IMX6_PCIE_FLAG_IMX6_PHY | - IMX6_PCIE_FLAG_IMX6_SPEED_CHANGE | - IMX6_PCIE_FLAG_SUPPORTS_SUSPEND, + .flags = IMX_PCIE_FLAG_IMX_PHY | + IMX_PCIE_FLAG_IMX_SPEED_CHANGE | + IMX_PCIE_FLAG_SUPPORTS_SUSPEND, .gpr = "fsl,imx6q-iomuxc-gpr", .clk_names = imx6sx_clks, .clks_cnt = ARRAY_SIZE(imx6sx_clks), @@ -1477,9 +1476,9 @@ static const struct imx6_pcie_drvdata drvdata[] = { }, [IMX6QP] = { .variant = IMX6QP, - .flags = IMX6_PCIE_FLAG_IMX6_PHY | - IMX6_PCIE_FLAG_IMX6_SPEED_CHANGE | - IMX6_PCIE_FLAG_SUPPORTS_SUSPEND, + .flags = IMX_PCIE_FLAG_IMX_PHY | + IMX_PCIE_FLAG_IMX_SPEED_CHANGE | + IMX_PCIE_FLAG_SUPPORTS_SUSPEND, .dbi_length = 0x200, .gpr = "fsl,imx6q-iomuxc-gpr", .clk_names = imx6q_clks, @@ -1488,13 +1487,13 @@ static const struct imx6_pcie_drvdata drvdata[] = { .ltssm_mask = IMX6Q_GPR12_PCIE_CTL_2, .mode_off[0] = IOMUXC_GPR12, .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE, - .init_phy = imx6_pcie_init_phy, + .init_phy = imx_pcie_init_phy, }, [IMX7D] = { .variant = IMX7D, - .flags = IMX6_PCIE_FLAG_SUPPORTS_SUSPEND | - IMX6_PCIE_FLAG_HAS_APP_RESET | - IMX6_PCIE_FLAG_HAS_PHY_RESET, + .flags = IMX_PCIE_FLAG_SUPPORTS_SUSPEND | + IMX_PCIE_FLAG_HAS_APP_RESET | + IMX_PCIE_FLAG_HAS_PHY_RESET, .gpr = "fsl,imx7d-iomuxc-gpr", .clk_names = imx6q_clks, .clks_cnt = ARRAY_SIZE(imx6q_clks), @@ -1504,8 +1503,8 @@ static const struct imx6_pcie_drvdata drvdata[] = { }, [IMX8MQ] = { .variant = IMX8MQ, - .flags = IMX6_PCIE_FLAG_HAS_APP_RESET | - IMX6_PCIE_FLAG_HAS_PHY_RESET, + .flags = IMX_PCIE_FLAG_HAS_APP_RESET | + IMX_PCIE_FLAG_HAS_PHY_RESET, .gpr = "fsl,imx8mq-iomuxc-gpr", .clk_names = imx8mq_clks, .clks_cnt = ARRAY_SIZE(imx8mq_clks), @@ -1517,9 +1516,9 @@ static const struct imx6_pcie_drvdata drvdata[] = { }, [IMX8MM] = { .variant = IMX8MM, - .flags = IMX6_PCIE_FLAG_SUPPORTS_SUSPEND | - IMX6_PCIE_FLAG_HAS_PHYDRV | - IMX6_PCIE_FLAG_HAS_APP_RESET, + .flags = IMX_PCIE_FLAG_SUPPORTS_SUSPEND | + IMX_PCIE_FLAG_HAS_PHYDRV | + IMX_PCIE_FLAG_HAS_APP_RESET, .gpr = "fsl,imx8mm-iomuxc-gpr", .clk_names = imx8mm_clks, .clks_cnt = ARRAY_SIZE(imx8mm_clks), @@ -1528,9 +1527,9 @@ static const struct imx6_pcie_drvdata drvdata[] = { }, [IMX8MP] = { .variant = IMX8MP, - .flags = IMX6_PCIE_FLAG_SUPPORTS_SUSPEND | - IMX6_PCIE_FLAG_HAS_PHYDRV | - IMX6_PCIE_FLAG_HAS_APP_RESET, + .flags = IMX_PCIE_FLAG_SUPPORTS_SUSPEND | + IMX_PCIE_FLAG_HAS_PHYDRV | + IMX_PCIE_FLAG_HAS_APP_RESET, .gpr = "fsl,imx8mp-iomuxc-gpr", .clk_names = imx8mm_clks, .clks_cnt = ARRAY_SIZE(imx8mm_clks), @@ -1539,7 +1538,7 @@ static const struct imx6_pcie_drvdata drvdata[] = { }, [IMX95] = { .variant = IMX95, - .flags = IMX6_PCIE_FLAG_HAS_SERDES, + .flags = IMX_PCIE_FLAG_HAS_SERDES, .clk_names = imx8mq_clks, .clks_cnt = ARRAY_SIZE(imx8mq_clks), .ltssm_off = IMX95_PE0_GEN_CTRL_3, @@ -1550,8 +1549,8 @@ static const struct imx6_pcie_drvdata drvdata[] = { }, [IMX8MQ_EP] = { .variant = IMX8MQ_EP, - .flags = IMX6_PCIE_FLAG_HAS_APP_RESET | - IMX6_PCIE_FLAG_HAS_PHY_RESET, + .flags = IMX_PCIE_FLAG_HAS_APP_RESET | + IMX_PCIE_FLAG_HAS_PHY_RESET, .mode = DW_PCIE_EP_TYPE, .gpr = "fsl,imx8mq-iomuxc-gpr", .clk_names = imx8mq_clks, @@ -1565,8 +1564,8 @@ static const struct imx6_pcie_drvdata drvdata[] = { }, [IMX8MM_EP] = { .variant = IMX8MM_EP, - .flags = IMX6_PCIE_FLAG_HAS_APP_RESET | - IMX6_PCIE_FLAG_HAS_PHYDRV, + .flags = IMX_PCIE_FLAG_HAS_APP_RESET | + IMX_PCIE_FLAG_HAS_PHYDRV, .mode = DW_PCIE_EP_TYPE, .gpr = "fsl,imx8mm-iomuxc-gpr", .clk_names = imx8mm_clks, @@ -1577,8 +1576,8 @@ static const struct imx6_pcie_drvdata drvdata[] = { }, [IMX8MP_EP] = { .variant = IMX8MP_EP, - .flags = IMX6_PCIE_FLAG_HAS_APP_RESET | - IMX6_PCIE_FLAG_HAS_PHYDRV, + .flags = IMX_PCIE_FLAG_HAS_APP_RESET | + IMX_PCIE_FLAG_HAS_PHYDRV, .mode = DW_PCIE_EP_TYPE, .gpr = "fsl,imx8mp-iomuxc-gpr", .clk_names = imx8mm_clks, @@ -1589,8 +1588,8 @@ static const struct imx6_pcie_drvdata drvdata[] = { }, [IMX95_EP] = { .variant = IMX95_EP, - .flags = IMX6_PCIE_FLAG_HAS_SERDES | - IMX6_PCIE_FLAG_SUPPORT_64BIT, + .flags = IMX_PCIE_FLAG_HAS_SERDES | + IMX_PCIE_FLAG_SUPPORT_64BIT, .clk_names = imx8mq_clks, .clks_cnt = ARRAY_SIZE(imx8mq_clks), .ltssm_off = IMX95_PE0_GEN_CTRL_3, @@ -1603,7 +1602,7 @@ static const struct imx6_pcie_drvdata drvdata[] = { }, }; -static const struct of_device_id imx6_pcie_of_match[] = { +static const struct of_device_id imx_pcie_of_match[] = { { .compatible = "fsl,imx6q-pcie", .data = &drvdata[IMX6Q], }, { .compatible = "fsl,imx6sx-pcie", .data = &drvdata[IMX6SX], }, { .compatible = "fsl,imx6qp-pcie", .data = &drvdata[IMX6QP], }, @@ -1619,19 +1618,19 @@ static const struct of_device_id imx6_pcie_of_match[] = { {}, }; -static struct platform_driver imx6_pcie_driver = { +static struct platform_driver imx_pcie_driver = { .driver = { .name = "imx6q-pcie", - .of_match_table = imx6_pcie_of_match, + .of_match_table = imx_pcie_of_match, .suppress_bind_attrs = true, - .pm = &imx6_pcie_pm_ops, + .pm = &imx_pcie_pm_ops, .probe_type = PROBE_PREFER_ASYNCHRONOUS, }, - .probe = imx6_pcie_probe, - .shutdown = imx6_pcie_shutdown, + .probe = imx_pcie_probe, + .shutdown = imx_pcie_shutdown, }; -static void imx6_pcie_quirk(struct pci_dev *dev) +static void imx_pcie_quirk(struct pci_dev *dev) { struct pci_bus *bus = dev->bus; struct dw_pcie_rp *pp = bus->sysdata; @@ -1641,33 +1640,33 @@ static void imx6_pcie_quirk(struct pci_dev *dev) return; /* Make sure we only quirk devices associated with this driver */ - if (bus->dev.parent->parent->driver != &imx6_pcie_driver.driver) + if (bus->dev.parent->parent->driver != &imx_pcie_driver.driver) return; if (pci_is_root_bus(bus)) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); - struct imx6_pcie *imx6_pcie = to_imx6_pcie(pci); + struct imx_pcie *imx_pcie = to_imx_pcie(pci); /* * Limit config length to avoid the kernel reading beyond * the register set and causing an abort on i.MX 6Quad */ - if (imx6_pcie->drvdata->dbi_length) { - dev->cfg_size = imx6_pcie->drvdata->dbi_length; + if (imx_pcie->drvdata->dbi_length) { + dev->cfg_size = imx_pcie->drvdata->dbi_length; dev_info(&dev->dev, "Limiting cfg_size to %d\n", dev->cfg_size); } } } DECLARE_PCI_FIXUP_CLASS_HEADER(PCI_VENDOR_ID_SYNOPSYS, 0xabcd, - PCI_CLASS_BRIDGE_PCI, 8, imx6_pcie_quirk); + PCI_CLASS_BRIDGE_PCI, 8, imx_pcie_quirk); -static int __init imx6_pcie_init(void) +static int __init imx_pcie_init(void) { #ifdef CONFIG_ARM struct device_node *np; - np = of_find_matching_node(NULL, imx6_pcie_of_match); + np = of_find_matching_node(NULL, imx_pcie_of_match); if (!np) return -ENODEV; of_node_put(np); @@ -1683,6 +1682,6 @@ static int __init imx6_pcie_init(void) "external abort on non-linefetch"); #endif - return platform_driver_register(&imx6_pcie_driver); + return platform_driver_register(&imx_pcie_driver); } -device_initcall(imx6_pcie_init); +device_initcall(imx_pcie_init); From 256867b74625a56ce1ff4bd89440c3fbbb357d18 Mon Sep 17 00:00:00 2001 From: Frank Li Date: Mon, 29 Jul 2024 16:18:12 -0400 Subject: [PATCH 189/302] PCI: imx6: Introduce SoC specific callbacks for controlling REFCLK MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of using the switch case statement to enable/disable the reference clock handled by this driver itself, let's introduce a new callback enable_ref_clk() and define it for platforms that require it. This simplifies the code. Link: https://lore.kernel.org/linux-pci/20240729-pci2_upstream-v8-5-b68ee5ef2b4d@nxp.com Signed-off-by: Frank Li [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Manivannan Sadhasivam --- drivers/pci/controller/dwc/pci-imx6.c | 111 ++++++++++++-------------- 1 file changed, 51 insertions(+), 60 deletions(-) diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c index 2f418508ed9b..d843ffb7cecb 100644 --- a/drivers/pci/controller/dwc/pci-imx6.c +++ b/drivers/pci/controller/dwc/pci-imx6.c @@ -101,6 +101,7 @@ struct imx_pcie_drvdata { const u32 mode_mask[IMX_PCIE_MAX_INSTANCES]; const struct pci_epc_features *epc_features; int (*init_phy)(struct imx_pcie *pcie); + int (*enable_ref_clk)(struct imx_pcie *pcie, bool enable); }; struct imx_pcie { @@ -582,21 +583,20 @@ static int imx_pcie_attach_pd(struct device *dev) return 0; } -static int imx_pcie_enable_ref_clk(struct imx_pcie *imx_pcie) +static int imx6sx_pcie_enable_ref_clk(struct imx_pcie *imx_pcie, bool enable) { - unsigned int offset; - int ret = 0; + if (enable) + regmap_clear_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12, + IMX6SX_GPR12_PCIE_TEST_POWERDOWN); - switch (imx_pcie->drvdata->variant) { - case IMX6SX: - regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12, - IMX6SX_GPR12_PCIE_TEST_POWERDOWN, 0); - break; - case IMX6QP: - case IMX6Q: + return 0; +} + +static int imx6q_pcie_enable_ref_clk(struct imx_pcie *imx_pcie, bool enable) +{ + if (enable) { /* power up core phy and enable ref clock */ - regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, - IMX6Q_GPR1_PCIE_TEST_PD, 0 << 18); + regmap_clear_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, IMX6Q_GPR1_PCIE_TEST_PD); /* * the async reset input need ref clock to sync internally, * when the ref clock comes after reset, internal synced @@ -604,55 +604,33 @@ static int imx_pcie_enable_ref_clk(struct imx_pcie *imx_pcie) * add one ~10us delay here. */ usleep_range(10, 100); - regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, - IMX6Q_GPR1_PCIE_REF_CLK_EN, 1 << 16); - break; - case IMX7D: - case IMX95: - case IMX95_EP: - break; - case IMX8MM: - case IMX8MM_EP: - case IMX8MQ: - case IMX8MQ_EP: - case IMX8MP: - case IMX8MP_EP: - offset = imx_pcie_grp_offset(imx_pcie); - /* - * Set the over ride low and enabled - * make sure that REF_CLK is turned on. - */ - regmap_update_bits(imx_pcie->iomuxc_gpr, offset, - IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE, - 0); - regmap_update_bits(imx_pcie->iomuxc_gpr, offset, - IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE_EN, - IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE_EN); - break; + regmap_set_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, IMX6Q_GPR1_PCIE_REF_CLK_EN); + } else { + regmap_clear_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, IMX6Q_GPR1_PCIE_REF_CLK_EN); + regmap_set_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, IMX6Q_GPR1_PCIE_TEST_PD); } - return ret; + return 0; } -static void imx_pcie_disable_ref_clk(struct imx_pcie *imx_pcie) +static int imx8mm_pcie_enable_ref_clk(struct imx_pcie *imx_pcie, bool enable) { - switch (imx_pcie->drvdata->variant) { - case IMX6QP: - case IMX6Q: - regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, - IMX6Q_GPR1_PCIE_REF_CLK_EN, 0); - regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, - IMX6Q_GPR1_PCIE_TEST_PD, - IMX6Q_GPR1_PCIE_TEST_PD); - break; - case IMX7D: - regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12, - IMX7D_GPR12_PCIE_PHY_REFCLK_SEL, - IMX7D_GPR12_PCIE_PHY_REFCLK_SEL); - break; - default: - break; + int offset = imx_pcie_grp_offset(imx_pcie); + + if (enable) { + regmap_clear_bits(imx_pcie->iomuxc_gpr, offset, IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE); + regmap_set_bits(imx_pcie->iomuxc_gpr, offset, IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE_EN); } + + return 0; +} + +static int imx7d_pcie_enable_ref_clk(struct imx_pcie *imx_pcie, bool enable) +{ + if (!enable) + regmap_set_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12, + IMX7D_GPR12_PCIE_PHY_REFCLK_SEL); + return 0; } static int imx_pcie_clk_enable(struct imx_pcie *imx_pcie) @@ -665,10 +643,12 @@ static int imx_pcie_clk_enable(struct imx_pcie *imx_pcie) if (ret) return ret; - ret = imx_pcie_enable_ref_clk(imx_pcie); - if (ret) { - dev_err(dev, "unable to enable pcie ref clock\n"); - goto err_ref_clk; + if (imx_pcie->drvdata->enable_ref_clk) { + ret = imx_pcie->drvdata->enable_ref_clk(imx_pcie, true); + if (ret) { + dev_err(dev, "Failed to enable PCIe REFCLK\n"); + goto err_ref_clk; + } } /* allow the clocks to stabilize */ @@ -683,7 +663,8 @@ err_ref_clk: static void imx_pcie_clk_disable(struct imx_pcie *imx_pcie) { - imx_pcie_disable_ref_clk(imx_pcie); + if (imx_pcie->drvdata->enable_ref_clk) + imx_pcie->drvdata->enable_ref_clk(imx_pcie, false); clk_bulk_disable_unprepare(imx_pcie->drvdata->clks_cnt, imx_pcie->clks); } @@ -1459,6 +1440,7 @@ static const struct imx_pcie_drvdata drvdata[] = { .mode_off[0] = IOMUXC_GPR12, .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE, .init_phy = imx_pcie_init_phy, + .enable_ref_clk = imx6q_pcie_enable_ref_clk, }, [IMX6SX] = { .variant = IMX6SX, @@ -1473,6 +1455,7 @@ static const struct imx_pcie_drvdata drvdata[] = { .mode_off[0] = IOMUXC_GPR12, .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE, .init_phy = imx6sx_pcie_init_phy, + .enable_ref_clk = imx6sx_pcie_enable_ref_clk, }, [IMX6QP] = { .variant = IMX6QP, @@ -1488,6 +1471,7 @@ static const struct imx_pcie_drvdata drvdata[] = { .mode_off[0] = IOMUXC_GPR12, .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE, .init_phy = imx_pcie_init_phy, + .enable_ref_clk = imx6q_pcie_enable_ref_clk, }, [IMX7D] = { .variant = IMX7D, @@ -1500,6 +1484,7 @@ static const struct imx_pcie_drvdata drvdata[] = { .mode_off[0] = IOMUXC_GPR12, .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE, .init_phy = imx7d_pcie_init_phy, + .enable_ref_clk = imx7d_pcie_enable_ref_clk, }, [IMX8MQ] = { .variant = IMX8MQ, @@ -1513,6 +1498,7 @@ static const struct imx_pcie_drvdata drvdata[] = { .mode_off[1] = IOMUXC_GPR12, .mode_mask[1] = IMX8MQ_GPR12_PCIE2_CTRL_DEVICE_TYPE, .init_phy = imx8mq_pcie_init_phy, + .enable_ref_clk = imx8mm_pcie_enable_ref_clk, }, [IMX8MM] = { .variant = IMX8MM, @@ -1524,6 +1510,7 @@ static const struct imx_pcie_drvdata drvdata[] = { .clks_cnt = ARRAY_SIZE(imx8mm_clks), .mode_off[0] = IOMUXC_GPR12, .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE, + .enable_ref_clk = imx8mm_pcie_enable_ref_clk, }, [IMX8MP] = { .variant = IMX8MP, @@ -1535,6 +1522,7 @@ static const struct imx_pcie_drvdata drvdata[] = { .clks_cnt = ARRAY_SIZE(imx8mm_clks), .mode_off[0] = IOMUXC_GPR12, .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE, + .enable_ref_clk = imx8mm_pcie_enable_ref_clk, }, [IMX95] = { .variant = IMX95, @@ -1561,6 +1549,7 @@ static const struct imx_pcie_drvdata drvdata[] = { .mode_mask[1] = IMX8MQ_GPR12_PCIE2_CTRL_DEVICE_TYPE, .epc_features = &imx8m_pcie_epc_features, .init_phy = imx8mq_pcie_init_phy, + .enable_ref_clk = imx8mm_pcie_enable_ref_clk, }, [IMX8MM_EP] = { .variant = IMX8MM_EP, @@ -1573,6 +1562,7 @@ static const struct imx_pcie_drvdata drvdata[] = { .mode_off[0] = IOMUXC_GPR12, .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE, .epc_features = &imx8m_pcie_epc_features, + .enable_ref_clk = imx8mm_pcie_enable_ref_clk, }, [IMX8MP_EP] = { .variant = IMX8MP_EP, @@ -1585,6 +1575,7 @@ static const struct imx_pcie_drvdata drvdata[] = { .mode_off[0] = IOMUXC_GPR12, .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE, .epc_features = &imx8m_pcie_epc_features, + .enable_ref_clk = imx8mm_pcie_enable_ref_clk, }, [IMX95_EP] = { .variant = IMX95_EP, From 5223084d1383a9b5a72989e173fb8b566766f49f Mon Sep 17 00:00:00 2001 From: Frank Li Date: Mon, 29 Jul 2024 16:18:13 -0400 Subject: [PATCH 190/302] PCI: imx6: Simplify switch-case logic by involve core_reset callback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of using the switch case statement to assert/dassert the core reset handled by this driver itself, let's introduce a new callback core_reset() and define it for platforms that require it. This simplifies the code. Link: https://lore.kernel.org/linux-pci/20240729-pci2_upstream-v8-5-b68ee5ef2b4d@nxp.com Signed-off-by: Frank Li [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Manivannan Sadhasivam --- drivers/pci/controller/dwc/pci-imx6.c | 134 ++++++++++++++------------ 1 file changed, 71 insertions(+), 63 deletions(-) diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c index d843ffb7cecb..a63975bd21ed 100644 --- a/drivers/pci/controller/dwc/pci-imx6.c +++ b/drivers/pci/controller/dwc/pci-imx6.c @@ -102,6 +102,7 @@ struct imx_pcie_drvdata { const struct pci_epc_features *epc_features; int (*init_phy)(struct imx_pcie *pcie); int (*enable_ref_clk)(struct imx_pcie *pcie, bool enable); + int (*core_reset)(struct imx_pcie *pcie, bool assert); }; struct imx_pcie { @@ -668,35 +669,75 @@ static void imx_pcie_clk_disable(struct imx_pcie *imx_pcie) clk_bulk_disable_unprepare(imx_pcie->drvdata->clks_cnt, imx_pcie->clks); } +static int imx6sx_pcie_core_reset(struct imx_pcie *imx_pcie, bool assert) +{ + if (assert) + regmap_set_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12, + IMX6SX_GPR12_PCIE_TEST_POWERDOWN); + + /* Force PCIe PHY reset */ + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR5, IMX6SX_GPR5_PCIE_BTNRST_RESET, + assert ? IMX6SX_GPR5_PCIE_BTNRST_RESET : 0); + return 0; +} + +static int imx6qp_pcie_core_reset(struct imx_pcie *imx_pcie, bool assert) +{ + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, IMX6Q_GPR1_PCIE_SW_RST, + assert ? IMX6Q_GPR1_PCIE_SW_RST : 0); + if (!assert) + usleep_range(200, 500); + + return 0; +} + +static int imx6q_pcie_core_reset(struct imx_pcie *imx_pcie, bool assert) +{ + if (!assert) + return 0; + + regmap_set_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, IMX6Q_GPR1_PCIE_TEST_PD); + regmap_set_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, IMX6Q_GPR1_PCIE_REF_CLK_EN); + + return 0; +} + +static int imx7d_pcie_core_reset(struct imx_pcie *imx_pcie, bool assert) +{ + struct dw_pcie *pci = imx_pcie->pci; + struct device *dev = pci->dev; + + if (assert) + return 0; + + /* + * Workaround for ERR010728, failure of PCI-e PLL VCO to + * oscillate, especially when cold. This turns off "Duty-cycle + * Corrector" and other mysterious undocumented things. + */ + + if (likely(imx_pcie->phy_base)) { + /* De-assert DCC_FB_EN */ + writel(PCIE_PHY_CMN_REG4_DCC_FB_EN, imx_pcie->phy_base + PCIE_PHY_CMN_REG4); + /* Assert RX_EQS and RX_EQS_SEL */ + writel(PCIE_PHY_CMN_REG24_RX_EQ_SEL | PCIE_PHY_CMN_REG24_RX_EQ, + imx_pcie->phy_base + PCIE_PHY_CMN_REG24); + /* Assert ATT_MODE */ + writel(PCIE_PHY_CMN_REG26_ATT_MODE, imx_pcie->phy_base + PCIE_PHY_CMN_REG26); + } else { + dev_warn(dev, "Unable to apply ERR010728 workaround. DT missing fsl,imx7d-pcie-phy phandle ?\n"); + } + imx7d_pcie_wait_for_phy_pll_lock(imx_pcie); + return 0; +} + static void imx_pcie_assert_core_reset(struct imx_pcie *imx_pcie) { reset_control_assert(imx_pcie->pciephy_reset); reset_control_assert(imx_pcie->apps_reset); - switch (imx_pcie->drvdata->variant) { - case IMX6SX: - regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12, - IMX6SX_GPR12_PCIE_TEST_POWERDOWN, - IMX6SX_GPR12_PCIE_TEST_POWERDOWN); - /* Force PCIe PHY reset */ - regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR5, - IMX6SX_GPR5_PCIE_BTNRST_RESET, - IMX6SX_GPR5_PCIE_BTNRST_RESET); - break; - case IMX6QP: - regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, - IMX6Q_GPR1_PCIE_SW_RST, - IMX6Q_GPR1_PCIE_SW_RST); - break; - case IMX6Q: - regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, - IMX6Q_GPR1_PCIE_TEST_PD, 1 << 18); - regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, - IMX6Q_GPR1_PCIE_REF_CLK_EN, 0 << 16); - break; - default: - break; - } + if (imx_pcie->drvdata->core_reset) + imx_pcie->drvdata->core_reset(imx_pcie, true); /* Some boards don't have PCIe reset GPIO. */ gpiod_set_value_cansleep(imx_pcie->reset_gpiod, 1); @@ -704,47 +745,10 @@ static void imx_pcie_assert_core_reset(struct imx_pcie *imx_pcie) static int imx_pcie_deassert_core_reset(struct imx_pcie *imx_pcie) { - struct dw_pcie *pci = imx_pcie->pci; - struct device *dev = pci->dev; - reset_control_deassert(imx_pcie->pciephy_reset); - switch (imx_pcie->drvdata->variant) { - case IMX7D: - /* Workaround for ERR010728, failure of PCI-e PLL VCO to - * oscillate, especially when cold. This turns off "Duty-cycle - * Corrector" and other mysterious undocumented things. - */ - if (likely(imx_pcie->phy_base)) { - /* De-assert DCC_FB_EN */ - writel(PCIE_PHY_CMN_REG4_DCC_FB_EN, - imx_pcie->phy_base + PCIE_PHY_CMN_REG4); - /* Assert RX_EQS and RX_EQS_SEL */ - writel(PCIE_PHY_CMN_REG24_RX_EQ_SEL - | PCIE_PHY_CMN_REG24_RX_EQ, - imx_pcie->phy_base + PCIE_PHY_CMN_REG24); - /* Assert ATT_MODE */ - writel(PCIE_PHY_CMN_REG26_ATT_MODE, - imx_pcie->phy_base + PCIE_PHY_CMN_REG26); - } else { - dev_warn(dev, "Unable to apply ERR010728 workaround. DT missing fsl,imx7d-pcie-phy phandle ?\n"); - } - - imx7d_pcie_wait_for_phy_pll_lock(imx_pcie); - break; - case IMX6SX: - regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR5, - IMX6SX_GPR5_PCIE_BTNRST_RESET, 0); - break; - case IMX6QP: - regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, - IMX6Q_GPR1_PCIE_SW_RST, 0); - - usleep_range(200, 500); - break; - default: - break; - } + if (imx_pcie->drvdata->core_reset) + imx_pcie->drvdata->core_reset(imx_pcie, false); /* Some boards don't have PCIe reset GPIO. */ if (imx_pcie->reset_gpiod) { @@ -1441,6 +1445,7 @@ static const struct imx_pcie_drvdata drvdata[] = { .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE, .init_phy = imx_pcie_init_phy, .enable_ref_clk = imx6q_pcie_enable_ref_clk, + .core_reset = imx6q_pcie_core_reset, }, [IMX6SX] = { .variant = IMX6SX, @@ -1456,6 +1461,7 @@ static const struct imx_pcie_drvdata drvdata[] = { .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE, .init_phy = imx6sx_pcie_init_phy, .enable_ref_clk = imx6sx_pcie_enable_ref_clk, + .core_reset = imx6sx_pcie_core_reset, }, [IMX6QP] = { .variant = IMX6QP, @@ -1472,6 +1478,7 @@ static const struct imx_pcie_drvdata drvdata[] = { .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE, .init_phy = imx_pcie_init_phy, .enable_ref_clk = imx6q_pcie_enable_ref_clk, + .core_reset = imx6qp_pcie_core_reset, }, [IMX7D] = { .variant = IMX7D, @@ -1485,6 +1492,7 @@ static const struct imx_pcie_drvdata drvdata[] = { .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE, .init_phy = imx7d_pcie_init_phy, .enable_ref_clk = imx7d_pcie_enable_ref_clk, + .core_reset = imx7d_pcie_core_reset, }, [IMX8MQ] = { .variant = IMX8MQ, From 52ac41b5cd60e5182a5cd33d1ad44622c3d1cc05 Mon Sep 17 00:00:00 2001 From: Frank Li Date: Mon, 29 Jul 2024 16:18:14 -0400 Subject: [PATCH 191/302] PCI: imx6: Improve comment for workaround ERR010728 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Improve comment about workaround ERR010728 by using official errata document content, see: https://www.nxp.com/webapp/Download?colCode=IMX7DS_2N09P Link: https://lore.kernel.org/linux-pci/20240729-pci2_upstream-v8-7-b68ee5ef2b4d@nxp.com Signed-off-by: Frank Li [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Manivannan Sadhasivam --- drivers/pci/controller/dwc/pci-imx6.c | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c index a63975bd21ed..0b165b6aa757 100644 --- a/drivers/pci/controller/dwc/pci-imx6.c +++ b/drivers/pci/controller/dwc/pci-imx6.c @@ -711,9 +711,26 @@ static int imx7d_pcie_core_reset(struct imx_pcie *imx_pcie, bool assert) return 0; /* - * Workaround for ERR010728, failure of PCI-e PLL VCO to - * oscillate, especially when cold. This turns off "Duty-cycle - * Corrector" and other mysterious undocumented things. + * Workaround for ERR010728 (IMX7DS_2N09P, Rev. 1.1, 4/2023): + * + * PCIe: PLL may fail to lock under corner conditions. + * + * Initial VCO oscillation may fail under corner conditions such as + * cold temperature which will cause the PCIe PLL fail to lock in the + * initialization phase. + * + * The Duty-cycle Corrector calibration must be disabled. + * + * 1. De-assert the G_RST signal by clearing + * SRC_PCIEPHY_RCR[PCIEPHY_G_RST]. + * 2. De-assert DCC_FB_EN by writing data “0x29” to the register + * address 0x306d0014 (PCIE_PHY_CMN_REG4). + * 3. Assert RX_EQS, RX_EQ_SEL by writing data “0x48” to the register + * address 0x306d0090 (PCIE_PHY_CMN_REG24). + * 4. Assert ATT_MODE by writing data “0xbc” to the register + * address 0x306d0098 (PCIE_PHY_CMN_REG26). + * 5. De-assert the CMN_RST signal by clearing register bit + * SRC_PCIEPHY_RCR[PCIEPHY_BTN] */ if (likely(imx_pcie->phy_base)) { From eea9ecebe2f8efed04a2ff67c2c9651f5cae571a Mon Sep 17 00:00:00 2001 From: Frank Li Date: Mon, 29 Jul 2024 16:18:15 -0400 Subject: [PATCH 192/302] PCI: imx6: Consolidate redundant if-checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consolidated redundant if-checks pertaining to imx_pcie->phy. Instead of two separate checks, merged them into one to improve code readability. Link: https://lore.kernel.org/linux-pci/20240729-pci2_upstream-v8-8-b68ee5ef2b4d@nxp.com Signed-off-by: Frank Li [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Manivannan Sadhasivam --- drivers/pci/controller/dwc/pci-imx6.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c index 0b165b6aa757..332d9b26f3f5 100644 --- a/drivers/pci/controller/dwc/pci-imx6.c +++ b/drivers/pci/controller/dwc/pci-imx6.c @@ -948,9 +948,7 @@ static int imx_pcie_host_init(struct dw_pcie_rp *pp) dev_err(dev, "pcie PHY power up failed\n"); goto err_clk_disable; } - } - if (imx_pcie->phy) { ret = phy_power_on(imx_pcie->phy); if (ret) { dev_err(dev, "waiting for PHY ready timeout!\n"); From 4f1e478f75e9818983ce30b98ca6d2d8182952eb Mon Sep 17 00:00:00 2001 From: Richard Zhu Date: Mon, 29 Jul 2024 16:18:16 -0400 Subject: [PATCH 193/302] dt-bindings: PCI: imx6q-pcie: Add i.MX8Q PCIe compatible string MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add i.MX8Q PCIe "fsl,imx8q-pcie" compatible strings. clock-names align DesignWare Cores (DWC) common naming convension. Link: https://lore.kernel.org/linux-pci/20240729-pci2_upstream-v8-9-b68ee5ef2b4d@nxp.com Signed-off-by: Richard Zhu Signed-off-by: Frank Li [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Manivannan Sadhasivam Reviewed-by: Rob Herring (Arm) Acked-by: Conor Dooley Acked-by: Manivannan Sadhasivam --- .../devicetree/bindings/pci/fsl,imx6q-pcie.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie.yaml b/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie.yaml index 8b8d77b1154b..1e05c560d797 100644 --- a/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie.yaml +++ b/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie.yaml @@ -30,6 +30,7 @@ properties: - fsl,imx8mm-pcie - fsl,imx8mp-pcie - fsl,imx95-pcie + - fsl,imx8q-pcie clocks: minItems: 3 @@ -184,6 +185,21 @@ allOf: - const: pcie_bus - const: pcie_aux + - if: + properties: + compatible: + enum: + - fsl,imx8q-pcie + then: + properties: + clocks: + maxItems: 3 + clock-names: + items: + - const: dbi + - const: mstr + - const: slv + unevaluatedProperties: false examples: From 8026f2d8e8a95a638a6cb83858bc2bdeed60a865 Mon Sep 17 00:00:00 2001 From: Frank Li Date: Mon, 29 Jul 2024 16:18:17 -0400 Subject: [PATCH 194/302] PCI: imx6: Call common PHY API to set mode, speed, and submode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Invoke the common PHY API to configure mode, speed, and submode. While these functions are optional in the PHY interface, they are necessary for certain PHY drivers. Lack of support for these functions in a PHY driver does not cause harm. Link: https://lore.kernel.org/linux-pci/20240729-pci2_upstream-v8-10-b68ee5ef2b4d@nxp.com Signed-off-by: Frank Li Signed-off-by: Krzysztof Wilczyński Reviewed-by: Manivannan Sadhasivam --- drivers/pci/controller/dwc/pci-imx6.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c index 332d9b26f3f5..f99d65a281ca 100644 --- a/drivers/pci/controller/dwc/pci-imx6.c +++ b/drivers/pci/controller/dwc/pci-imx6.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -226,6 +227,10 @@ static void imx_pcie_configure_type(struct imx_pcie *imx_pcie) id = imx_pcie->controller_id; + /* If mode_mask is 0, then generic PHY driver is used to set the mode */ + if (!drvdata->mode_mask[0]) + return; + /* If mode_mask[id] is zero, means each controller have its individual gpr */ if (!drvdata->mode_mask[id]) id = 0; @@ -801,7 +806,11 @@ static void imx_pcie_ltssm_enable(struct device *dev) { struct imx_pcie *imx_pcie = dev_get_drvdata(dev); const struct imx_pcie_drvdata *drvdata = imx_pcie->drvdata; + u8 offset = dw_pcie_find_capability(imx_pcie->pci, PCI_CAP_ID_EXP); + u32 tmp; + tmp = dw_pcie_readl_dbi(imx_pcie->pci, offset + PCI_EXP_LNKCAP); + phy_set_speed(imx_pcie->phy, FIELD_GET(PCI_EXP_LNKCAP_SLS, tmp)); if (drvdata->ltssm_mask) regmap_update_bits(imx_pcie->iomuxc_gpr, drvdata->ltssm_off, drvdata->ltssm_mask, drvdata->ltssm_mask); @@ -814,6 +823,7 @@ static void imx_pcie_ltssm_disable(struct device *dev) struct imx_pcie *imx_pcie = dev_get_drvdata(dev); const struct imx_pcie_drvdata *drvdata = imx_pcie->drvdata; + phy_set_speed(imx_pcie->phy, 0); if (drvdata->ltssm_mask) regmap_update_bits(imx_pcie->iomuxc_gpr, drvdata->ltssm_off, drvdata->ltssm_mask, 0); @@ -949,6 +959,12 @@ static int imx_pcie_host_init(struct dw_pcie_rp *pp) goto err_clk_disable; } + ret = phy_set_mode_ext(imx_pcie->phy, PHY_MODE_PCIE, PHY_MODE_PCIE_RC); + if (ret) { + dev_err(dev, "unable to set PCIe PHY mode\n"); + goto err_phy_exit; + } + ret = phy_power_on(imx_pcie->phy); if (ret) { dev_err(dev, "waiting for PHY ready timeout!\n"); From 2a0091f9419cb6dbbada3a4c8d9e86117b80ead4 Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Mon, 2 Sep 2024 15:28:59 -0500 Subject: [PATCH 195/302] PCI: brcmstb: Sort enums, pcie_offsets[], pcie_cfg_data, .compatible strings Sort enum pcie_soc_base values. Rename pcie_offsets_bmips_7425[] to pcie_offsets_bcm7425[] to match BCM7425 pcie_soc_base enum, bcm7425_cfg, and "brcm,bcm7425-pcie" .compatible string. Rename pcie_offset_bcm7278[] to pcie_offsets_bcm7278[] to match other "pcie_offsets" names. Rename pcie_offset_bcm7712[] to pcie_offsets_bcm7712[] to match other "pcie_offsets" names. Sort pcie_offsets_*[] by SoC name, move them all together, indent values for easy reading. Sort pcie_cfg_data structs by SoC name. Sort .compatible strings by SoC name. No functional change intended. Link: https://lore.kernel.org/r/20240902205456.227409-1-helgaas@kernel.org Signed-off-by: Bjorn Helgaas Reviewed-by: Florian Fainelli --- drivers/pci/controller/pcie-brcmstb.c | 114 +++++++++++++------------- 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/controller/pcie-brcmstb.c index 2bd8ff0f875c..9bfbd90e14ac 100644 --- a/drivers/pci/controller/pcie-brcmstb.c +++ b/drivers/pci/controller/pcie-brcmstb.c @@ -220,11 +220,11 @@ enum { enum pcie_soc_base { GENERIC, - BCM7425, - BCM7435, + BCM2711, BCM4908, BCM7278, - BCM2711, + BCM7425, + BCM7435, BCM7712, }; @@ -1665,26 +1665,34 @@ static void brcm_pcie_remove(struct platform_device *pdev) } static const int pcie_offsets[] = { - [RGR1_SW_INIT_1] = 0x9210, - [EXT_CFG_INDEX] = 0x9000, - [EXT_CFG_DATA] = 0x9004, - [PCIE_HARD_DEBUG] = 0x4204, - [PCIE_INTR2_CPU_BASE] = 0x4300, + [RGR1_SW_INIT_1] = 0x9210, + [EXT_CFG_INDEX] = 0x9000, + [EXT_CFG_DATA] = 0x9004, + [PCIE_HARD_DEBUG] = 0x4204, + [PCIE_INTR2_CPU_BASE] = 0x4300, }; -static const int pcie_offsets_bmips_7425[] = { - [RGR1_SW_INIT_1] = 0x8010, - [EXT_CFG_INDEX] = 0x8300, - [EXT_CFG_DATA] = 0x8304, - [PCIE_HARD_DEBUG] = 0x4204, - [PCIE_INTR2_CPU_BASE] = 0x4300, +static const int pcie_offsets_bcm7278[] = { + [RGR1_SW_INIT_1] = 0xc010, + [EXT_CFG_INDEX] = 0x9000, + [EXT_CFG_DATA] = 0x9004, + [PCIE_HARD_DEBUG] = 0x4204, + [PCIE_INTR2_CPU_BASE] = 0x4300, }; -static const int pcie_offset_bcm7712[] = { - [EXT_CFG_INDEX] = 0x9000, - [EXT_CFG_DATA] = 0x9004, - [PCIE_HARD_DEBUG] = 0x4304, - [PCIE_INTR2_CPU_BASE] = 0x4400, +static const int pcie_offsets_bcm7425[] = { + [RGR1_SW_INIT_1] = 0x8010, + [EXT_CFG_INDEX] = 0x8300, + [EXT_CFG_DATA] = 0x8304, + [PCIE_HARD_DEBUG] = 0x4204, + [PCIE_INTR2_CPU_BASE] = 0x4300, +}; + +static const int pcie_offsets_bcm7712[] = { + [EXT_CFG_INDEX] = 0x9000, + [EXT_CFG_DATA] = 0x9004, + [PCIE_HARD_DEBUG] = 0x4304, + [PCIE_INTR2_CPU_BASE] = 0x4400, }; static const struct pcie_cfg_data generic_cfg = { @@ -1695,8 +1703,32 @@ static const struct pcie_cfg_data generic_cfg = { .num_inbound_wins = 3, }; +static const struct pcie_cfg_data bcm2711_cfg = { + .offsets = pcie_offsets, + .soc_base = BCM2711, + .perst_set = brcm_pcie_perst_set_generic, + .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, + .num_inbound_wins = 3, +}; + +static const struct pcie_cfg_data bcm4908_cfg = { + .offsets = pcie_offsets, + .soc_base = BCM4908, + .perst_set = brcm_pcie_perst_set_4908, + .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, + .num_inbound_wins = 3, +}; + +static const struct pcie_cfg_data bcm7278_cfg = { + .offsets = pcie_offsets_bcm7278, + .soc_base = BCM7278, + .perst_set = brcm_pcie_perst_set_7278, + .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_7278, + .num_inbound_wins = 3, +}; + static const struct pcie_cfg_data bcm7425_cfg = { - .offsets = pcie_offsets_bmips_7425, + .offsets = pcie_offsets_bcm7425, .soc_base = BCM7425, .perst_set = brcm_pcie_perst_set_generic, .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, @@ -1711,40 +1743,8 @@ static const struct pcie_cfg_data bcm7435_cfg = { .num_inbound_wins = 3, }; -static const struct pcie_cfg_data bcm4908_cfg = { - .offsets = pcie_offsets, - .soc_base = BCM4908, - .perst_set = brcm_pcie_perst_set_4908, - .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, - .num_inbound_wins = 3, -}; - -static const int pcie_offset_bcm7278[] = { - [RGR1_SW_INIT_1] = 0xc010, - [EXT_CFG_INDEX] = 0x9000, - [EXT_CFG_DATA] = 0x9004, - [PCIE_HARD_DEBUG] = 0x4204, - [PCIE_INTR2_CPU_BASE] = 0x4300, -}; - -static const struct pcie_cfg_data bcm7278_cfg = { - .offsets = pcie_offset_bcm7278, - .soc_base = BCM7278, - .perst_set = brcm_pcie_perst_set_7278, - .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_7278, - .num_inbound_wins = 3, -}; - -static const struct pcie_cfg_data bcm2711_cfg = { - .offsets = pcie_offsets, - .soc_base = BCM2711, - .perst_set = brcm_pcie_perst_set_generic, - .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, - .num_inbound_wins = 3, -}; - static const struct pcie_cfg_data bcm7216_cfg = { - .offsets = pcie_offset_bcm7278, + .offsets = pcie_offsets_bcm7278, .soc_base = BCM7278, .perst_set = brcm_pcie_perst_set_7278, .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_7278, @@ -1753,7 +1753,7 @@ static const struct pcie_cfg_data bcm7216_cfg = { }; static const struct pcie_cfg_data bcm7712_cfg = { - .offsets = pcie_offset_bcm7712, + .offsets = pcie_offsets_bcm7712, .perst_set = brcm_pcie_perst_set_7278, .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, .soc_base = BCM7712, @@ -1764,11 +1764,11 @@ static const struct of_device_id brcm_pcie_match[] = { { .compatible = "brcm,bcm2711-pcie", .data = &bcm2711_cfg }, { .compatible = "brcm,bcm4908-pcie", .data = &bcm4908_cfg }, { .compatible = "brcm,bcm7211-pcie", .data = &generic_cfg }, - { .compatible = "brcm,bcm7278-pcie", .data = &bcm7278_cfg }, { .compatible = "brcm,bcm7216-pcie", .data = &bcm7216_cfg }, - { .compatible = "brcm,bcm7445-pcie", .data = &generic_cfg }, - { .compatible = "brcm,bcm7435-pcie", .data = &bcm7435_cfg }, + { .compatible = "brcm,bcm7278-pcie", .data = &bcm7278_cfg }, { .compatible = "brcm,bcm7425-pcie", .data = &bcm7425_cfg }, + { .compatible = "brcm,bcm7435-pcie", .data = &bcm7435_cfg }, + { .compatible = "brcm,bcm7445-pcie", .data = &generic_cfg }, { .compatible = "brcm,bcm7712-pcie", .data = &bcm7712_cfg }, {}, }; From d591f6804e7e1310881c9224d72247a2b65039af Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Tue, 27 Aug 2024 18:48:46 -0500 Subject: [PATCH 196/302] PCI: Wait for device readiness with Configuration RRS After a device reset, delays are required before the device can successfully complete config accesses. PCIe r6.0, sec 6.6, specifies some delays required before software can perform config accesses. Devices that require more time after those delays may respond to config accesses with Configuration Request Retry Status (RRS) completions. Callers of pci_dev_wait() are responsible for delays until the device can respond to config accesses. pci_dev_wait() waits any additional time until the device can successfully complete config accesses. Reading config space of devices that are not present or not ready typically returns ~0 (PCI_ERROR_RESPONSE). Previously we polled the Command register until we got a value other than ~0. This is sometimes a problem because Root Complex handling of RRS completions may include several retries and implementation-specific behavior that is invisible to software (see sec 2.3.2), so the exponential backoff in pci_dev_wait() may not work as intended. Linux enables Configuration RRS Software Visibility on all Root Ports that support it. If it is enabled, read the Vendor ID instead of the Command register. RRS completions cause immediate return of the 0x0001 reserved Vendor ID value, so the pci_dev_wait() backoff works correctly. When a read of Vendor ID eventually completes successfully by returning a non-0x0001 value (the Vendor ID or 0xffff for VFs), the device should be initialized and ready to respond to config requests. For conventional PCI devices or devices below Root Ports that don't support Configuration RRS Software Visibility, poll the Command register as before. This was developed independently, but is very similar to Stanislav Spassov's previous work at https://lore.kernel.org/linux-pci/20200223122057.6504-1-stanspas@amazon.com Link: https://lore.kernel.org/r/20240827234848.4429-2-helgaas@kernel.org Signed-off-by: Bjorn Helgaas Tested-by: Duc Dang --- drivers/pci/pci.c | 41 ++++++++++++++++++++++++++++------------- drivers/pci/pci.h | 5 +++++ drivers/pci/probe.c | 9 +++------ include/linux/pci.h | 1 + 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index e3a49f66982d..fc2ecb7fe081 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -1283,7 +1283,9 @@ static int pci_dev_wait(struct pci_dev *dev, char *reset_type, int timeout) { int delay = 1; bool retrain = false; - struct pci_dev *bridge; + struct pci_dev *root, *bridge; + + root = pcie_find_root_port(dev); if (pci_is_pcie(dev)) { bridge = pci_upstream_bridge(dev); @@ -1292,16 +1294,23 @@ static int pci_dev_wait(struct pci_dev *dev, char *reset_type, int timeout) } /* - * After reset, the device should not silently discard config - * requests, but it may still indicate that it needs more time by - * responding to them with CRS completions. The Root Port will - * generally synthesize ~0 (PCI_ERROR_RESPONSE) data to complete - * the read (except when CRS SV is enabled and the read was for the - * Vendor ID; in that case it synthesizes 0x0001 data). + * The caller has already waited long enough after a reset that the + * device should respond to config requests, but it may respond + * with Request Retry Status (RRS) if it needs more time to + * initialize. * - * Wait for the device to return a non-CRS completion. Read the - * Command register instead of Vendor ID so we don't have to - * contend with the CRS SV value. + * If the device is below a Root Port with Configuration RRS + * Software Visibility enabled, reading the Vendor ID returns a + * special data value if the device responded with RRS. Read the + * Vendor ID until we get non-RRS status. + * + * If there's no Root Port or Configuration RRS Software Visibility + * is not enabled, the device may still respond with RRS, but + * hardware may retry the config request. If no retries receive + * Successful Completion, hardware generally synthesizes ~0 + * (PCI_ERROR_RESPONSE) data to complete the read. Reading Vendor + * ID for VFs and non-existent devices also returns ~0, so read the + * Command register until it returns something other than ~0. */ for (;;) { u32 id; @@ -1311,9 +1320,15 @@ static int pci_dev_wait(struct pci_dev *dev, char *reset_type, int timeout) return -ENOTTY; } - pci_read_config_dword(dev, PCI_COMMAND, &id); - if (!PCI_POSSIBLE_ERROR(id)) - break; + if (root && root->config_crs_sv) { + pci_read_config_dword(dev, PCI_VENDOR_ID, &id); + if (!pci_bus_crs_vendor_id(id)) + break; + } else { + pci_read_config_dword(dev, PCI_COMMAND, &id); + if (!PCI_POSSIBLE_ERROR(id)) + break; + } if (delay > timeout) { pci_warn(dev, "not ready %dms after %s; giving up\n", diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 79c8398f3938..fa1997bc2667 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -139,6 +139,11 @@ bool pci_bridge_d3_possible(struct pci_dev *dev); void pci_bridge_d3_update(struct pci_dev *dev); int pci_bridge_wait_for_secondary_bus(struct pci_dev *dev, char *reset_type); +static inline bool pci_bus_crs_vendor_id(u32 l) +{ + return (l & 0xffff) == PCI_VENDOR_ID_PCI_SIG; +} + static inline void pci_wakeup_event(struct pci_dev *dev) { /* Wait 100 ms before the system can be put into a sleep state. */ diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index b14b9876c030..b1615da9eb6b 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -1209,9 +1209,11 @@ static void pci_enable_crs(struct pci_dev *pdev) /* Enable CRS Software Visibility if supported */ pcie_capability_read_word(pdev, PCI_EXP_RTCAP, &root_cap); - if (root_cap & PCI_EXP_RTCAP_CRSVIS) + if (root_cap & PCI_EXP_RTCAP_CRSVIS) { pcie_capability_set_word(pdev, PCI_EXP_RTCTL, PCI_EXP_RTCTL_CRSSVE); + pdev->config_crs_sv = 1; + } } static unsigned int pci_scan_child_bus_extend(struct pci_bus *bus, @@ -2343,11 +2345,6 @@ struct pci_dev *pci_alloc_dev(struct pci_bus *bus) } EXPORT_SYMBOL(pci_alloc_dev); -static bool pci_bus_crs_vendor_id(u32 l) -{ - return (l & 0xffff) == PCI_VENDOR_ID_PCI_SIG; -} - static bool pci_bus_wait_crs(struct pci_bus *bus, int devfn, u32 *l, int timeout) { diff --git a/include/linux/pci.h b/include/linux/pci.h index 4cf89a4b4cbc..121d8d94d6d0 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -371,6 +371,7 @@ struct pci_dev { can be generated */ unsigned int pme_poll:1; /* Poll device's PME status bit */ unsigned int pinned:1; /* Whether this dev is pinned */ + unsigned int config_crs_sv:1; /* Config CRS software visibility */ unsigned int imm_ready:1; /* Supports Immediate Readiness */ unsigned int d1_support:1; /* Low power state D1 is supported */ unsigned int d2_support:1; /* Low power state D2 is supported */ From dd4e47eab886abf28859ccf8aad373983015f89e Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Tue, 27 Aug 2024 18:48:47 -0500 Subject: [PATCH 197/302] PCI: aardvark: Correct Configuration RRS checking Per PCIe r6.0, sec 2.3.2, when a Root Complex handles a Completion with Request Retry Status for a Configuration Read Request that includes both bytes of the Vendor ID field, it must complete the Request to the host by returning 0001h for the Vendor ID and all 1's for any additional bytes. Previously we only returned the 0001h Vendor ID value if we got an RRS completion for reads of exactly 4 bytes. A read of 2 bytes would not qualify, although the spec says it should. Check for reads of 2 or more bytes including the Vendor ID. I don't think this will fix any observable problems because RRS only applies to the first config reads after reset, and those are all currently dword (4-byte) reads. Link: https://lore.kernel.org/r/20240827234848.4429-3-helgaas@kernel.org Signed-off-by: Bjorn Helgaas --- drivers/pci/controller/pci-aardvark.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/pci/controller/pci-aardvark.c b/drivers/pci/controller/pci-aardvark.c index 8b3e1a079cf3..e66594558ce2 100644 --- a/drivers/pci/controller/pci-aardvark.c +++ b/drivers/pci/controller/pci-aardvark.c @@ -1153,11 +1153,11 @@ static int advk_pcie_rd_conf(struct pci_bus *bus, u32 devfn, size, val); /* - * Completion Retry Status is possible to return only when reading all - * 4 bytes from PCI_VENDOR_ID and PCI_DEVICE_ID registers at once and - * CRSSVE flag on Root Bridge is enabled. + * Completion Retry Status is possible to return only when reading + * both bytes from PCI_VENDOR_ID at once and CRSSVE flag on Root + * Port is enabled. */ - allow_crs = (where == PCI_VENDOR_ID) && (size == 4) && + allow_crs = (where == PCI_VENDOR_ID) && (size >= 2) && (le16_to_cpu(pcie->bridge.pcie_conf.rootctl) & PCI_EXP_RTCTL_CRSSVE); From 87f10faf166a9114aa0d4132298cad379de16fdd Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Tue, 27 Aug 2024 18:48:48 -0500 Subject: [PATCH 198/302] PCI: Rename CRS Completion Status to RRS PCIe r6.0 changed the abbreviation for "Configuration Request Retry Status" Completion Status from "CRS" to "RRS" and uses the terminology of "Configuration RRS Software Visibility" instead of "CRS Software Visibility". Align the Linux usage with the r6.0 spec language. No functional change intended. It's confusing to make this change, but I think "RRS" *is* a better abbreviation because it was easy to interpret "CRS" as "Completion Retry Status", which really didn't make any sense. Link: https://lore.kernel.org/r/20240827234848.4429-4-helgaas@kernel.org Signed-off-by: Bjorn Helgaas --- drivers/bcma/driver_pci_host.c | 10 ++-- drivers/pci/controller/dwc/pcie-tegra194.c | 18 +++--- drivers/pci/controller/pci-aardvark.c | 64 +++++++++++----------- drivers/pci/controller/pci-xgene.c | 6 +- drivers/pci/controller/pcie-iproc.c | 18 +++--- drivers/pci/pci-bridge-emul.c | 4 +- drivers/pci/pci.c | 4 +- drivers/pci/pci.h | 8 +-- drivers/pci/probe.c | 28 +++++----- include/linux/bcma/bcma_driver_pci.h | 2 +- include/linux/pci.h | 2 +- include/uapi/linux/pci_regs.h | 6 +- 12 files changed, 86 insertions(+), 84 deletions(-) diff --git a/drivers/bcma/driver_pci_host.c b/drivers/bcma/driver_pci_host.c index ed3be52ab63d..8540052d37c5 100644 --- a/drivers/bcma/driver_pci_host.c +++ b/drivers/bcma/driver_pci_host.c @@ -334,7 +334,7 @@ static u8 bcma_find_pci_capability(struct bcma_drv_pci *pc, unsigned int dev, } /* If the root port is capable of returning Config Request - * Retry Status (CRS) Completion Status to software then + * Retry Status (RRS) Completion Status to software then * enable the feature. */ static void bcma_core_pci_enable_crs(struct bcma_drv_pci *pc) @@ -348,10 +348,10 @@ static void bcma_core_pci_enable_crs(struct bcma_drv_pci *pc) NULL); root_cap = cap_ptr + PCI_EXP_RTCAP; bcma_extpci_read_config(pc, 0, 0, root_cap, &val16, sizeof(u16)); - if (val16 & BCMA_CORE_PCI_RC_CRS_VISIBILITY) { - /* Enable CRS software visibility */ + if (val16 & BCMA_CORE_PCI_RC_RRS_VISIBILITY) { + /* Enable Configuration RRS Software Visibility */ root_ctrl = cap_ptr + PCI_EXP_RTCTL; - val16 = PCI_EXP_RTCTL_CRSSVE; + val16 = PCI_EXP_RTCTL_RRS_SVE; bcma_extpci_read_config(pc, 0, 0, root_ctrl, &val16, sizeof(u16)); @@ -360,7 +360,7 @@ static void bcma_core_pci_enable_crs(struct bcma_drv_pci *pc) * 100 ms wait time from the end of Reset. If the device is * not done with its internal initialization, it must at * least return a completion TLP, with a completion status - * of "Configuration Request Retry Status (CRS)". The root + * of "Configuration Request Retry Status (RRS)". The root * complex must complete the request to the host by returning * a read-data value of 0001h for the Vendor ID field and * all 1s for any additional bytes included in the request. diff --git a/drivers/pci/controller/dwc/pcie-tegra194.c b/drivers/pci/controller/dwc/pcie-tegra194.c index 4bf7b433417a..886354342ef1 100644 --- a/drivers/pci/controller/dwc/pcie-tegra194.c +++ b/drivers/pci/controller/dwc/pcie-tegra194.c @@ -183,11 +183,11 @@ #define GEN3_EQ_CONTROL_OFF_FB_MODE_MASK GENMASK(3, 0) #define PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT 0x8D0 -#define AMBA_ERROR_RESPONSE_CRS_SHIFT 3 -#define AMBA_ERROR_RESPONSE_CRS_MASK GENMASK(1, 0) -#define AMBA_ERROR_RESPONSE_CRS_OKAY 0 -#define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFFFFFF 1 -#define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 2 +#define AMBA_ERROR_RESPONSE_RRS_SHIFT 3 +#define AMBA_ERROR_RESPONSE_RRS_MASK GENMASK(1, 0) +#define AMBA_ERROR_RESPONSE_RRS_OKAY 0 +#define AMBA_ERROR_RESPONSE_RRS_OKAY_FFFFFFFF 1 +#define AMBA_ERROR_RESPONSE_RRS_OKAY_FFFF0001 2 #define MSIX_ADDR_MATCH_LOW_OFF 0x940 #define MSIX_ADDR_MATCH_LOW_OFF_EN BIT(0) @@ -907,11 +907,11 @@ static int tegra_pcie_dw_host_init(struct dw_pcie_rp *pp) dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0); - /* Enable as 0xFFFF0001 response for CRS */ + /* Enable as 0xFFFF0001 response for RRS */ val = dw_pcie_readl_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT); - val &= ~(AMBA_ERROR_RESPONSE_CRS_MASK << AMBA_ERROR_RESPONSE_CRS_SHIFT); - val |= (AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 << - AMBA_ERROR_RESPONSE_CRS_SHIFT); + val &= ~(AMBA_ERROR_RESPONSE_RRS_MASK << AMBA_ERROR_RESPONSE_RRS_SHIFT); + val |= (AMBA_ERROR_RESPONSE_RRS_OKAY_FFFF0001 << + AMBA_ERROR_RESPONSE_RRS_SHIFT); dw_pcie_writel_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT, val); /* Clear Slot Clock Configuration bit if SRNS configuration */ diff --git a/drivers/pci/controller/pci-aardvark.c b/drivers/pci/controller/pci-aardvark.c index e66594558ce2..afccae3bfbc6 100644 --- a/drivers/pci/controller/pci-aardvark.c +++ b/drivers/pci/controller/pci-aardvark.c @@ -50,7 +50,7 @@ #define PIO_COMPLETION_STATUS_MASK GENMASK(9, 7) #define PIO_COMPLETION_STATUS_OK 0 #define PIO_COMPLETION_STATUS_UR 1 -#define PIO_COMPLETION_STATUS_CRS 2 +#define PIO_COMPLETION_STATUS_RRS 2 #define PIO_COMPLETION_STATUS_CA 4 #define PIO_NON_POSTED_REQ BIT(10) #define PIO_ERR_STATUS BIT(11) @@ -262,7 +262,7 @@ enum { #define MSI_IRQ_NUM 32 -#define CFG_RD_CRS_VAL 0xffff0001 +#define CFG_RD_RRS_VAL 0xffff0001 struct advk_pcie { struct platform_device *pdev; @@ -649,7 +649,7 @@ static void advk_pcie_setup_hw(struct advk_pcie *pcie) advk_pcie_train_link(pcie); } -static int advk_pcie_check_pio_status(struct advk_pcie *pcie, bool allow_crs, u32 *val) +static int advk_pcie_check_pio_status(struct advk_pcie *pcie, bool allow_rrs, u32 *val) { struct device *dev = &pcie->pdev->dev; u32 reg; @@ -669,7 +669,7 @@ static int advk_pcie_check_pio_status(struct advk_pcie *pcie, bool allow_crs, u3 * 2) value Unsupported Request(1) of COMPLETION_STATUS(bit9:7) only * means a PIO write error, and for PIO read it is successful with * a read value of 0xFFFFFFFF. - * 3) value Completion Retry Status(CRS) of COMPLETION_STATUS(bit9:7) + * 3) value Config Request Retry Status(RRS) of COMPLETION_STATUS(bit9:7) * only means a PIO write error, and for PIO read it is successful * with a read value of 0xFFFF0001. * 4) value Completer Abort (CA) of COMPLETION_STATUS(bit9:7) means @@ -694,10 +694,10 @@ static int advk_pcie_check_pio_status(struct advk_pcie *pcie, bool allow_crs, u3 strcomp_status = "UR"; ret = -EOPNOTSUPP; break; - case PIO_COMPLETION_STATUS_CRS: - if (allow_crs && val) { - /* PCIe r4.0, sec 2.3.2, says: - * If CRS Software Visibility is enabled: + case PIO_COMPLETION_STATUS_RRS: + if (allow_rrs && val) { + /* PCIe r6.0, sec 2.3.2, says: + * If Configuration RRS Software Visibility is enabled: * For a Configuration Read Request that includes both * bytes of the Vendor ID field of a device Function's * Configuration Space Header, the Root Complex must @@ -706,22 +706,22 @@ static int advk_pcie_check_pio_status(struct advk_pcie *pcie, bool allow_crs, u3 * all '1's for any additional bytes included in the * request. * - * So CRS in this case is not an error status. + * So RRS in this case is not an error status. */ - *val = CFG_RD_CRS_VAL; + *val = CFG_RD_RRS_VAL; strcomp_status = NULL; ret = 0; break; } - /* PCIe r4.0, sec 2.3.2, says: - * If CRS Software Visibility is not enabled, the Root Complex + /* PCIe r6.0, sec 2.3.2, says: + * If RRS Software Visibility is not enabled, the Root Complex * must re-issue the Configuration Request as a new Request. - * If CRS Software Visibility is enabled: For a Configuration + * If RRS Software Visibility is enabled: For a Configuration * Write Request or for any other Configuration Read Request, * the Root Complex must re-issue the Configuration Request as * a new Request. * A Root Complex implementation may choose to limit the number - * of Configuration Request/CRS Completion Status loops before + * of Configuration Request/RRS Completion Status loops before * determining that something is wrong with the target of the * Request and taking appropriate action, e.g., complete the * Request to the host as a failed transaction. @@ -729,7 +729,7 @@ static int advk_pcie_check_pio_status(struct advk_pcie *pcie, bool allow_crs, u3 * So return -EAGAIN and caller (pci-aardvark.c driver) will * re-issue request again up to the PIO_RETRY_CNT retries. */ - strcomp_status = "CRS"; + strcomp_status = "RRS"; ret = -EAGAIN; break; case PIO_COMPLETION_STATUS_CA: @@ -920,8 +920,8 @@ advk_pci_bridge_emul_pcie_conf_write(struct pci_bridge_emul *bridge, case PCI_EXP_RTCTL: { u16 rootctl = le16_to_cpu(bridge->pcie_conf.rootctl); - /* Only emulation of PMEIE and CRSSVE bits is provided */ - rootctl &= PCI_EXP_RTCTL_PMEIE | PCI_EXP_RTCTL_CRSSVE; + /* Only emulation of PMEIE and RRS_SVE bits is provided */ + rootctl &= PCI_EXP_RTCTL_PMEIE | PCI_EXP_RTCTL_RRS_SVE; bridge->pcie_conf.rootctl = cpu_to_le16(rootctl); break; } @@ -1075,7 +1075,7 @@ static int advk_sw_pci_bridge_init(struct advk_pcie *pcie) bridge->pcie_conf.slotsta = cpu_to_le16(PCI_EXP_SLTSTA_PDS); /* Indicates supports for Completion Retry Status */ - bridge->pcie_conf.rootcap = cpu_to_le16(PCI_EXP_RTCAP_CRSVIS); + bridge->pcie_conf.rootcap = cpu_to_le16(PCI_EXP_RTCAP_RRS_SV); bridge->subsystem_vendor_id = advk_readl(pcie, PCIE_CORE_SSDEV_ID_REG) & 0xffff; bridge->subsystem_id = advk_readl(pcie, PCIE_CORE_SSDEV_ID_REG) >> 16; @@ -1141,7 +1141,7 @@ static int advk_pcie_rd_conf(struct pci_bus *bus, u32 devfn, { struct advk_pcie *pcie = bus->sysdata; int retry_count; - bool allow_crs; + bool allow_rrs; u32 reg; int ret; @@ -1153,16 +1153,16 @@ static int advk_pcie_rd_conf(struct pci_bus *bus, u32 devfn, size, val); /* - * Completion Retry Status is possible to return only when reading - * both bytes from PCI_VENDOR_ID at once and CRSSVE flag on Root - * Port is enabled. + * Configuration Request Retry Status (RRS) is possible to return + * only when reading both bytes from PCI_VENDOR_ID at once and + * RRS_SVE flag on Root Port is enabled. */ - allow_crs = (where == PCI_VENDOR_ID) && (size >= 2) && + allow_rrs = (where == PCI_VENDOR_ID) && (size >= 2) && (le16_to_cpu(pcie->bridge.pcie_conf.rootctl) & - PCI_EXP_RTCTL_CRSSVE); + PCI_EXP_RTCTL_RRS_SVE); if (advk_pcie_pio_is_running(pcie)) - goto try_crs; + goto try_rrs; /* Program the control register */ reg = advk_readl(pcie, PIO_CTRL); @@ -1189,12 +1189,12 @@ static int advk_pcie_rd_conf(struct pci_bus *bus, u32 devfn, ret = advk_pcie_wait_pio(pcie); if (ret < 0) - goto try_crs; + goto try_rrs; retry_count += ret; /* Check PIO status and get the read result */ - ret = advk_pcie_check_pio_status(pcie, allow_crs, val); + ret = advk_pcie_check_pio_status(pcie, allow_rrs, val); } while (ret == -EAGAIN && retry_count < PIO_RETRY_CNT); if (ret < 0) @@ -1207,13 +1207,13 @@ static int advk_pcie_rd_conf(struct pci_bus *bus, u32 devfn, return PCIBIOS_SUCCESSFUL; -try_crs: +try_rrs: /* - * If it is possible, return Completion Retry Status so that caller - * tries to issue the request again instead of failing. + * If it is possible, return Configuration Request Retry Status so + * that caller tries to issue the request again instead of failing. */ - if (allow_crs) { - *val = CFG_RD_CRS_VAL; + if (allow_rrs) { + *val = CFG_RD_RRS_VAL; return PCIBIOS_SUCCESSFUL; } diff --git a/drivers/pci/controller/pci-xgene.c b/drivers/pci/controller/pci-xgene.c index 8e457fa450a2..1e2ebbfa36d1 100644 --- a/drivers/pci/controller/pci-xgene.c +++ b/drivers/pci/controller/pci-xgene.c @@ -171,17 +171,17 @@ static int xgene_pcie_config_read32(struct pci_bus *bus, unsigned int devfn, /* * The v1 controller has a bug in its Configuration Request Retry - * Status (CRS) logic: when CRS Software Visibility is enabled and + * Status (RRS) logic: when RRS Software Visibility is enabled and * we read the Vendor and Device ID of a non-existent device, the * controller fabricates return data of 0xFFFF0001 ("device exists * but is not ready") instead of 0xFFFFFFFF (PCI_ERROR_RESPONSE) * ("device does not exist"). This causes the PCI core to retry * the read until it times out. Avoid this by not claiming to - * support CRS SV. + * support RRS SV. */ if (pci_is_root_bus(bus) && (port->version == XGENE_PCIE_IP_VER_1) && ((where & ~0x3) == XGENE_V1_PCI_EXP_CAP + PCI_EXP_RTCTL)) - *val &= ~(PCI_EXP_RTCAP_CRSVIS << 16); + *val &= ~(PCI_EXP_RTCAP_RRS_SV << 16); if (size <= 2) *val = (*val >> (8 * (where & 3))) & ((1 << (size * 8)) - 1); diff --git a/drivers/pci/controller/pcie-iproc.c b/drivers/pci/controller/pcie-iproc.c index 97f739a2c9f8..22134e95574b 100644 --- a/drivers/pci/controller/pcie-iproc.c +++ b/drivers/pci/controller/pcie-iproc.c @@ -54,7 +54,7 @@ #define CFG_RD_SUCCESS 0 #define CFG_RD_UR 1 -#define CFG_RD_CRS 2 +#define CFG_RD_RRS 2 #define CFG_RD_CA 3 #define CFG_RETRY_STATUS 0xffff0001 #define CFG_RETRY_STATUS_TIMEOUT_US 500000 /* 500 milliseconds */ @@ -485,31 +485,31 @@ static unsigned int iproc_pcie_cfg_retry(struct iproc_pcie *pcie, u32 status; /* - * As per PCIe spec r3.1, sec 2.3.2, CRS Software Visibility only + * As per PCIe r6.0, sec 2.3.2, Config RRS Software Visibility only * affects config reads of the Vendor ID. For config writes or any * other config reads, the Root may automatically reissue the * configuration request again as a new request. * * For config reads, this hardware returns CFG_RETRY_STATUS data - * when it receives a CRS completion, regardless of the address of - * the read or the CRS Software Visibility Enable bit. As a + * when it receives a RRS completion, regardless of the address of + * the read or the RRS Software Visibility Enable bit. As a * partial workaround for this, we retry in software any read that * returns CFG_RETRY_STATUS. * * Note that a non-Vendor ID config register may have a value of * CFG_RETRY_STATUS. If we read that, we can't distinguish it from - * a CRS completion, so we will incorrectly retry the read and + * a RRS completion, so we will incorrectly retry the read and * eventually return the wrong data (0xffffffff). */ data = readl(cfg_data_p); while (data == CFG_RETRY_STATUS && timeout--) { /* - * CRS state is set in CFG_RD status register + * RRS state is set in CFG_RD status register * This will handle the case where CFG_RETRY_STATUS is * valid config data. */ status = iproc_pcie_read_reg(pcie, IPROC_PCIE_CFG_RD_STATUS); - if (status != CFG_RD_CRS) + if (status != CFG_RD_RRS) return data; udelay(1); @@ -556,8 +556,8 @@ static void iproc_pcie_fix_cap(struct iproc_pcie *pcie, int where, u32 *val) break; case IPROC_PCI_EXP_CAP + PCI_EXP_RTCTL: - /* Don't advertise CRS SV support */ - *val &= ~(PCI_EXP_RTCAP_CRSVIS << 16); + /* Don't advertise RRS SV support */ + *val &= ~(PCI_EXP_RTCAP_RRS_SV << 16); break; default: diff --git a/drivers/pci/pci-bridge-emul.c b/drivers/pci/pci-bridge-emul.c index 9334b2dd4764..6658c1edd464 100644 --- a/drivers/pci/pci-bridge-emul.c +++ b/drivers/pci/pci-bridge-emul.c @@ -257,8 +257,8 @@ struct pci_bridge_reg_behavior pcie_cap_regs_behavior[PCI_CAP_PCIE_SIZEOF / 4] = */ .rw = (PCI_EXP_RTCTL_SECEE | PCI_EXP_RTCTL_SENFEE | PCI_EXP_RTCTL_SEFEE | PCI_EXP_RTCTL_PMEIE | - PCI_EXP_RTCTL_CRSSVE), - .ro = PCI_EXP_RTCAP_CRSVIS << 16, + PCI_EXP_RTCTL_RRS_SVE), + .ro = PCI_EXP_RTCAP_RRS_SV << 16, }, [PCI_EXP_RTSTA / 4] = { diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index fc2ecb7fe081..55d6124acb2d 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -1320,9 +1320,9 @@ static int pci_dev_wait(struct pci_dev *dev, char *reset_type, int timeout) return -ENOTTY; } - if (root && root->config_crs_sv) { + if (root && root->config_rrs_sv) { pci_read_config_dword(dev, PCI_VENDOR_ID, &id); - if (!pci_bus_crs_vendor_id(id)) + if (!pci_bus_rrs_vendor_id(id)) break; } else { pci_read_config_dword(dev, PCI_COMMAND, &id); diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index fa1997bc2667..061cc1b48fd8 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -139,7 +139,7 @@ bool pci_bridge_d3_possible(struct pci_dev *dev); void pci_bridge_d3_update(struct pci_dev *dev); int pci_bridge_wait_for_secondary_bus(struct pci_dev *dev, char *reset_type); -static inline bool pci_bus_crs_vendor_id(u32 l) +static inline bool pci_bus_rrs_vendor_id(u32 l) { return (l & 0xffff) == PCI_VENDOR_ID_PCI_SIG; } @@ -295,10 +295,10 @@ void pci_put_host_bridge_device(struct device *dev); int pci_configure_extended_tags(struct pci_dev *dev, void *ign); bool pci_bus_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *pl, - int crs_timeout); + int rrs_timeout); bool pci_bus_generic_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *pl, - int crs_timeout); -int pci_idt_bus_quirk(struct pci_bus *bus, int devfn, u32 *pl, int crs_timeout); + int rrs_timeout); +int pci_idt_bus_quirk(struct pci_bus *bus, int devfn, u32 *pl, int rrs_timeout); int pci_setup_device(struct pci_dev *dev); int __pci_read_base(struct pci_dev *dev, enum pci_bar_type type, diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index b1615da9eb6b..e79d9bea32bf 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -1203,16 +1203,16 @@ struct pci_bus *pci_add_new_bus(struct pci_bus *parent, struct pci_dev *dev, } EXPORT_SYMBOL(pci_add_new_bus); -static void pci_enable_crs(struct pci_dev *pdev) +static void pci_enable_rrs_sv(struct pci_dev *pdev) { u16 root_cap = 0; - /* Enable CRS Software Visibility if supported */ + /* Enable Configuration RRS Software Visibility if supported */ pcie_capability_read_word(pdev, PCI_EXP_RTCAP, &root_cap); - if (root_cap & PCI_EXP_RTCAP_CRSVIS) { + if (root_cap & PCI_EXP_RTCAP_RRS_SV) { pcie_capability_set_word(pdev, PCI_EXP_RTCTL, - PCI_EXP_RTCTL_CRSSVE); - pdev->config_crs_sv = 1; + PCI_EXP_RTCTL_RRS_SVE); + pdev->config_rrs_sv = 1; } } @@ -1328,7 +1328,7 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev, pci_write_config_word(dev, PCI_BRIDGE_CONTROL, bctl & ~PCI_BRIDGE_CTL_MASTER_ABORT); - pci_enable_crs(dev); + pci_enable_rrs_sv(dev); if ((secondary || subordinate) && !pcibios_assign_all_busses() && !is_cardbus && !broken) { @@ -2345,23 +2345,23 @@ struct pci_dev *pci_alloc_dev(struct pci_bus *bus) } EXPORT_SYMBOL(pci_alloc_dev); -static bool pci_bus_wait_crs(struct pci_bus *bus, int devfn, u32 *l, +static bool pci_bus_wait_rrs(struct pci_bus *bus, int devfn, u32 *l, int timeout) { int delay = 1; - if (!pci_bus_crs_vendor_id(*l)) - return true; /* not a CRS completion */ + if (!pci_bus_rrs_vendor_id(*l)) + return true; /* not a Configuration RRS completion */ if (!timeout) - return false; /* CRS, but caller doesn't want to wait */ + return false; /* RRS, but caller doesn't want to wait */ /* * We got the reserved Vendor ID that indicates a completion with - * Configuration Request Retry Status (CRS). Retry until we get a + * Configuration Request Retry Status (RRS). Retry until we get a * valid Vendor ID or we time out. */ - while (pci_bus_crs_vendor_id(*l)) { + while (pci_bus_rrs_vendor_id(*l)) { if (delay > timeout) { pr_warn("pci %04x:%02x:%02x.%d: not ready after %dms; giving up\n", pci_domain_nr(bus), bus->number, @@ -2400,8 +2400,8 @@ bool pci_bus_generic_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l, *l == 0x0000ffff || *l == 0xffff0000) return false; - if (pci_bus_crs_vendor_id(*l)) - return pci_bus_wait_crs(bus, devfn, l, timeout); + if (pci_bus_rrs_vendor_id(*l)) + return pci_bus_wait_rrs(bus, devfn, l, timeout); return true; } diff --git a/include/linux/bcma/bcma_driver_pci.h b/include/linux/bcma/bcma_driver_pci.h index 68da8dba5162..dba41b65ae0d 100644 --- a/include/linux/bcma/bcma_driver_pci.h +++ b/include/linux/bcma/bcma_driver_pci.h @@ -203,7 +203,7 @@ struct pci_dev; #define BCMA_CORE_PCI_MDIO_RXCTRL0 0x840 /* PCIE Root Capability Register bits (Host mode only) */ -#define BCMA_CORE_PCI_RC_CRS_VISIBILITY 0x0001 +#define BCMA_CORE_PCI_RC_RRS_VISIBILITY 0x0001 struct bcma_drv_pci; struct bcma_bus; diff --git a/include/linux/pci.h b/include/linux/pci.h index 121d8d94d6d0..27d8ce977674 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -371,7 +371,7 @@ struct pci_dev { can be generated */ unsigned int pme_poll:1; /* Poll device's PME status bit */ unsigned int pinned:1; /* Whether this dev is pinned */ - unsigned int config_crs_sv:1; /* Config CRS software visibility */ + unsigned int config_rrs_sv:1; /* Config RRS software visibility */ unsigned int imm_ready:1; /* Supports Immediate Readiness */ unsigned int d1_support:1; /* Low power state D1 is supported */ unsigned int d2_support:1; /* Low power state D2 is supported */ diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h index 94c00996e633..f94591f9f5e9 100644 --- a/include/uapi/linux/pci_regs.h +++ b/include/uapi/linux/pci_regs.h @@ -634,9 +634,11 @@ #define PCI_EXP_RTCTL_SENFEE 0x0002 /* System Error on Non-Fatal Error */ #define PCI_EXP_RTCTL_SEFEE 0x0004 /* System Error on Fatal Error */ #define PCI_EXP_RTCTL_PMEIE 0x0008 /* PME Interrupt Enable */ -#define PCI_EXP_RTCTL_CRSSVE 0x0010 /* CRS Software Visibility Enable */ +#define PCI_EXP_RTCTL_RRS_SVE 0x0010 /* Config RRS Software Visibility Enable */ +#define PCI_EXP_RTCTL_CRSSVE PCI_EXP_RTCTL_RRS_SVE /* compatibility */ #define PCI_EXP_RTCAP 0x1e /* Root Capabilities */ -#define PCI_EXP_RTCAP_CRSVIS 0x0001 /* CRS Software Visibility capability */ +#define PCI_EXP_RTCAP_RRS_SV 0x0001 /* Config RRS Software Visibility */ +#define PCI_EXP_RTCAP_CRSVIS PCI_EXP_RTCAP_RRS_SV /* compatibility */ #define PCI_EXP_RTSTA 0x20 /* Root Status */ #define PCI_EXP_RTSTA_PME_RQ_ID 0x0000ffff /* PME Requester ID */ #define PCI_EXP_RTSTA_PME 0x00010000 /* PME status */ From c2699778e6be4757ee0b16449ab8777c6b46e6d0 Mon Sep 17 00:00:00 2001 From: Richard Zhu Date: Mon, 29 Jul 2024 16:18:18 -0400 Subject: [PATCH 199/302] PCI: imx6: Add i.MX8Q PCIe Root Complex (RC) support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement i.MX8Q (i.MX8QM, i.MX8QXP, and i.MX8DXL) PCIe Root Complex (RC) support. While the controller resembles that of i.MX8MP, the PHY differs significantly. Also, there's a distinction between PCI bus addresses and CPU addresses. Introduce IMX_PCIE_FLAG_CPU_ADDR_FIXUP in drvdata::flags to indicate driver need the cpu_addr_fixup() callback to facilitate CPU address to PCI bus address conversion according to "ranges" property. Link: https://lore.kernel.org/linux-pci/20240729-pci2_upstream-v8-11-b68ee5ef2b4d@nxp.com Signed-off-by: Richard Zhu Signed-off-by: Frank Li Signed-off-by: Krzysztof Wilczyński [bhelgaas: check resource_list_first_type() for NULL] Signed-off-by: Bjorn Helgaas Reviewed-by: Manivannan Sadhasivam --- drivers/pci/controller/dwc/pci-imx6.c | 28 +++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c index f99d65a281ca..5a40cd7f29d4 100644 --- a/drivers/pci/controller/dwc/pci-imx6.c +++ b/drivers/pci/controller/dwc/pci-imx6.c @@ -65,6 +65,7 @@ enum imx_pcie_variants { IMX8MQ, IMX8MM, IMX8MP, + IMX8Q, IMX95, IMX8MQ_EP, IMX8MM_EP, @@ -80,6 +81,7 @@ enum imx_pcie_variants { #define IMX_PCIE_FLAG_HAS_PHY_RESET BIT(5) #define IMX_PCIE_FLAG_HAS_SERDES BIT(6) #define IMX_PCIE_FLAG_SUPPORT_64BIT BIT(7) +#define IMX_PCIE_FLAG_CPU_ADDR_FIXUP BIT(8) #define imx_check_flag(pci, val) (pci->drvdata->flags & val) @@ -1010,6 +1012,22 @@ static void imx_pcie_host_exit(struct dw_pcie_rp *pp) regulator_disable(imx_pcie->vpcie); } +static u64 imx_pcie_cpu_addr_fixup(struct dw_pcie *pcie, u64 cpu_addr) +{ + struct imx_pcie *imx_pcie = to_imx_pcie(pcie); + struct dw_pcie_rp *pp = &pcie->pp; + struct resource_entry *entry; + + if (!(imx_pcie->drvdata->flags & IMX_PCIE_FLAG_CPU_ADDR_FIXUP)) + return cpu_addr; + + entry = resource_list_first_type(&pp->bridge->windows, IORESOURCE_MEM); + if (!entry) + return cpu_addr; + + return cpu_addr - entry->offset; +} + static const struct dw_pcie_host_ops imx_pcie_host_ops = { .init = imx_pcie_host_init, .deinit = imx_pcie_host_exit, @@ -1018,6 +1036,7 @@ static const struct dw_pcie_host_ops imx_pcie_host_ops = { static const struct dw_pcie_ops dw_pcie_ops = { .start_link = imx_pcie_start_link, .stop_link = imx_pcie_stop_link, + .cpu_addr_fixup = imx_pcie_cpu_addr_fixup, }; static void imx_pcie_ep_init(struct dw_pcie_ep *ep) @@ -1460,6 +1479,7 @@ static const char * const imx6q_clks[] = {"pcie_bus", "pcie", "pcie_phy"}; static const char * const imx8mm_clks[] = {"pcie_bus", "pcie", "pcie_aux"}; static const char * const imx8mq_clks[] = {"pcie_bus", "pcie", "pcie_phy", "pcie_aux"}; static const char * const imx6sx_clks[] = {"pcie_bus", "pcie", "pcie_phy", "pcie_inbound_axi"}; +static const char * const imx8q_clks[] = {"mstr", "slv", "dbi"}; static const struct imx_pcie_drvdata drvdata[] = { [IMX6Q] = { @@ -1563,6 +1583,13 @@ static const struct imx_pcie_drvdata drvdata[] = { .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE, .enable_ref_clk = imx8mm_pcie_enable_ref_clk, }, + [IMX8Q] = { + .variant = IMX8Q, + .flags = IMX_PCIE_FLAG_HAS_PHYDRV | + IMX_PCIE_FLAG_CPU_ADDR_FIXUP, + .clk_names = imx8q_clks, + .clks_cnt = ARRAY_SIZE(imx8q_clks), + }, [IMX95] = { .variant = IMX95, .flags = IMX_PCIE_FLAG_HAS_SERDES, @@ -1640,6 +1667,7 @@ static const struct of_device_id imx_pcie_of_match[] = { { .compatible = "fsl,imx8mq-pcie", .data = &drvdata[IMX8MQ], }, { .compatible = "fsl,imx8mm-pcie", .data = &drvdata[IMX8MM], }, { .compatible = "fsl,imx8mp-pcie", .data = &drvdata[IMX8MP], }, + { .compatible = "fsl,imx8q-pcie", .data = &drvdata[IMX8Q], }, { .compatible = "fsl,imx95-pcie", .data = &drvdata[IMX95], }, { .compatible = "fsl,imx8mq-pcie-ep", .data = &drvdata[IMX8MQ_EP], }, { .compatible = "fsl,imx8mm-pcie-ep", .data = &drvdata[IMX8MM_EP], }, From dff07b5e4bb8da01fc93786e6712af6de01eaf7d Mon Sep 17 00:00:00 2001 From: Matthew Gerlach Date: Wed, 17 Jul 2024 13:17:56 -0500 Subject: [PATCH 200/302] dt-bindings: PCI: altera: msi: Convert to YAML MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Convert the devicetree bindings for the Altera PCIe MSI controller from text to YAML. Link: https://lore.kernel.org/linux-pci/20240717181756.2177553-1-matthew.gerlach@linux.intel.com Signed-off-by: Matthew Gerlach [kwilczynski: remove unused msi0 label] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Conor Dooley --- .../bindings/pci/altera-pcie-msi.txt | 27 -------- .../bindings/pci/altr,msi-controller.yaml | 65 +++++++++++++++++++ MAINTAINERS | 2 +- 3 files changed, 66 insertions(+), 28 deletions(-) delete mode 100644 Documentation/devicetree/bindings/pci/altera-pcie-msi.txt create mode 100644 Documentation/devicetree/bindings/pci/altr,msi-controller.yaml diff --git a/Documentation/devicetree/bindings/pci/altera-pcie-msi.txt b/Documentation/devicetree/bindings/pci/altera-pcie-msi.txt deleted file mode 100644 index 9514c327d31b..000000000000 --- a/Documentation/devicetree/bindings/pci/altera-pcie-msi.txt +++ /dev/null @@ -1,27 +0,0 @@ -* Altera PCIe MSI controller - -Required properties: -- compatible: should contain "altr,msi-1.0" -- reg: specifies the physical base address of the controller and - the length of the memory mapped region. -- reg-names: must include the following entries: - "csr": CSR registers - "vector_slave": vectors slave port region -- interrupts: specifies the interrupt source of the parent interrupt - controller. The format of the interrupt specifier depends on the - parent interrupt controller. -- num-vectors: number of vectors, range 1 to 32. -- msi-controller: indicates that this is MSI controller node - - -Example -msi0: msi@0xFF200000 { - compatible = "altr,msi-1.0"; - reg = <0xFF200000 0x00000010 - 0xFF200010 0x00000080>; - reg-names = "csr", "vector_slave"; - interrupt-parent = <&hps_0_arm_gic_0>; - interrupts = <0 42 4>; - msi-controller; - num-vectors = <32>; -}; diff --git a/Documentation/devicetree/bindings/pci/altr,msi-controller.yaml b/Documentation/devicetree/bindings/pci/altr,msi-controller.yaml new file mode 100644 index 000000000000..98814862d006 --- /dev/null +++ b/Documentation/devicetree/bindings/pci/altr,msi-controller.yaml @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +# Copyright (C) 2015, 2024, Intel Corporation +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/altr,msi-controller.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Altera PCIe MSI controller + +maintainers: + - Matthew Gerlach + +properties: + compatible: + enum: + - altr,msi-1.0 + + reg: + items: + - description: CSR registers + - description: Vectors slave port region + + reg-names: + items: + - const: csr + - const: vector_slave + + interrupts: + maxItems: 1 + + msi-controller: true + + num-vectors: + description: number of vectors + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 32 + +required: + - compatible + - reg + - reg-names + - interrupts + - msi-controller + - num-vectors + +allOf: + - $ref: /schemas/interrupt-controller/msi-controller.yaml# + +unevaluatedProperties: false + +examples: + - | + #include + #include + msi@ff200000 { + compatible = "altr,msi-1.0"; + reg = <0xff200000 0x00000010>, + <0xff200010 0x00000080>; + reg-names = "csr", "vector_slave"; + interrupt-parent = <&hps_0_arm_gic_0>; + interrupts = ; + msi-controller; + num-vectors = <32>; + }; diff --git a/MAINTAINERS b/MAINTAINERS index 8ebc8a46fc0f..1bbdb0b49ab4 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -17602,7 +17602,7 @@ PCI MSI DRIVER FOR ALTERA MSI IP M: Joyce Ooi L: linux-pci@vger.kernel.org S: Supported -F: Documentation/devicetree/bindings/pci/altera-pcie-msi.txt +F: Documentation/devicetree/bindings/pci/altr,msi-controller.yaml F: drivers/pci/controller/pcie-altera-msi.c PCI MSI DRIVER FOR APPLIEDMICRO XGENE From f500a2f1282750fb344ce535d78071cf1493efd1 Mon Sep 17 00:00:00 2001 From: Richard Zhu Date: Tue, 13 Aug 2024 15:42:20 +0800 Subject: [PATCH 201/302] dt-bindings: PCI: imx6q-pcie: Add reg-name "dbi2" and "atu" for i.MX8M PCIe Endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add reg-name: "dbi2", "atu" for i.MX8M PCIe Endpoint. For i.MX8M PCIe EP, the dbi2 and atu addresses are pre-defined in the driver. This method is not good. In commit b7d67c6130ee ("PCI: imx6: Add iMX95 Endpoint (EP) support"), Frank suggests to fetch the dbi2 and atu from DT directly. This commit is preparation to do that for i.MX8M PCIe EP. These changes wouldn't break driver function. When "dbi2" and "atu" properties are present, i.MX PCIe driver would fetch the according base addresses from DT directly. If only two reg properties are provided, i.MX PCIe driver would fall back to the old method. Link: https://lore.kernel.org/linux-pci/1723534943-28499-2-git-send-email-hongxing.zhu@nxp.com Signed-off-by: Richard Zhu Signed-off-by: Krzysztof Wilczyński Reviewed-by: Frank Li Reviewed-by: Rob Herring (Arm) --- .../devicetree/bindings/pci/fsl,imx6q-pcie-ep.yaml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie-ep.yaml b/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie-ep.yaml index a06f75df8458..84ca12e8b25b 100644 --- a/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie-ep.yaml +++ b/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie-ep.yaml @@ -65,12 +65,14 @@ allOf: then: properties: reg: - minItems: 2 - maxItems: 2 + minItems: 4 + maxItems: 4 reg-names: items: - const: dbi - const: addr_space + - const: dbi2 + - const: atu - if: properties: @@ -129,8 +131,11 @@ examples: pcie_ep: pcie-ep@33800000 { compatible = "fsl,imx8mp-pcie-ep"; - reg = <0x33800000 0x000400000>, <0x18000000 0x08000000>; - reg-names = "dbi", "addr_space"; + reg = <0x33800000 0x100000>, + <0x18000000 0x8000000>, + <0x33900000 0x100000>, + <0x33b00000 0x100000>; + reg-names = "dbi", "addr_space", "dbi2", "atu"; clocks = <&clk IMX8MP_CLK_HSIO_ROOT>, <&clk IMX8MP_CLK_HSIO_AXI>, <&clk IMX8MP_CLK_PCIE_ROOT>; From 2910306655a7072640021563ec9501bfa67f0cb1 Mon Sep 17 00:00:00 2001 From: Alex Williamson Date: Thu, 12 Sep 2024 15:53:27 -0600 Subject: [PATCH 202/302] PCI: Mark Creative Labs EMU20k2 INTx masking as broken MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per user reports, the Creative Labs EMU20k2 (Sound Blaster X-Fi Titanium Series) generates spurious interrupts when used with vfio-pci unless DisINTx masking support is disabled. Thus, quirk the device to mark INTx masking as broken. Closes: https://lore.kernel.org/all/VI1PR10MB8207C507DB5420AB4C7281E0DB9A2@VI1PR10MB8207.EURPRD10.PROD.OUTLOOK.COM Link: https://lore.kernel.org/linux-pci/20240912215331.839220-1-alex.williamson@redhat.com Reported-by: zdravko delineshev Signed-off-by: Alex Williamson [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński --- drivers/pci/quirks.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index e085ec3d9982..e78cdad4d0e5 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -3608,6 +3608,8 @@ DECLARE_PCI_FIXUP_FINAL(0x1814, 0x0601, /* Ralink RT2800 802.11n PCI */ quirk_broken_intx_masking); DECLARE_PCI_FIXUP_FINAL(0x1b7c, 0x0004, /* Ceton InfiniTV4 */ quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CREATIVE, PCI_DEVICE_ID_CREATIVE_20K2, + quirk_broken_intx_masking); /* * Realtek RTL8169 PCI Gigabit Ethernet Controller (rev 10) From d3745e3ae6c0eec517d431be926742b6e8b9b64a Mon Sep 17 00:00:00 2001 From: Manivannan Sadhasivam Date: Fri, 30 Aug 2024 13:53:19 +0530 Subject: [PATCH 203/302] PCI: qcom-ep: Enable controller resources like PHY only after refclk is available MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit qcom_pcie_enable_resources() is called by qcom_pcie_ep_probe() and it enables the controller resources like clocks, regulator, PHY. On one of the new unreleased Qcom SoC, PHY enablement depends on the active refclk. And on all of the supported Qcom endpoint SoCs, refclk comes from the host (RC). So calling qcom_pcie_enable_resources() without refclk causes the NoC (Network On Chip) error in the endpoint SoC and in turn results in a whole SoC crash and rebooting into EDL (Emergency Download) mode which is an unrecoverable state. But qcom_pcie_enable_resources() is already called by qcom_pcie_perst_deassert() when PERST# is deasserted, and refclk is available at that time. Hence, remove the unnecessary call to qcom_pcie_enable_resources() from qcom_pcie_ep_probe() to prevent the above mentioned crash. It should be noted that this commit prevents the crash only under normal working condition (booting endpoint before host), but the crash may also occur if PERST# assert happens at the wrong time. For avoiding the crash completely, it is recommended to use SRIS mode which allows the endpoint SoC to generate its own refclk. The driver is not supporting SRIS mode currently, but will be added in the future. Fixes: 869bc5253406 ("PCI: dwc: ep: Fix DBI access failure for drivers requiring refclk from host") Link: https://lore.kernel.org/linux-pci/20240830082319.51387-1-manivannan.sadhasivam@linaro.org Tested-by: Dmitry Baryshkov Signed-off-by: Manivannan Sadhasivam Signed-off-by: Krzysztof Wilczyński --- drivers/pci/controller/dwc/pcie-qcom-ep.c | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/drivers/pci/controller/dwc/pcie-qcom-ep.c b/drivers/pci/controller/dwc/pcie-qcom-ep.c index d0a27fa6fdc8..3c1b695638f3 100644 --- a/drivers/pci/controller/dwc/pcie-qcom-ep.c +++ b/drivers/pci/controller/dwc/pcie-qcom-ep.c @@ -857,21 +857,15 @@ static int qcom_pcie_ep_probe(struct platform_device *pdev) if (ret) return ret; - ret = qcom_pcie_enable_resources(pcie_ep); - if (ret) { - dev_err(dev, "Failed to enable resources: %d\n", ret); - return ret; - } - ret = dw_pcie_ep_init(&pcie_ep->pci.ep); if (ret) { dev_err(dev, "Failed to initialize endpoint: %d\n", ret); - goto err_disable_resources; + return ret; } ret = qcom_pcie_ep_enable_irq_resources(pdev, pcie_ep); if (ret) - goto err_disable_resources; + goto err_ep_deinit; name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node); if (!name) { @@ -888,8 +882,8 @@ err_disable_irqs: disable_irq(pcie_ep->global_irq); disable_irq(pcie_ep->perst_irq); -err_disable_resources: - qcom_pcie_disable_resources(pcie_ep); +err_ep_deinit: + dw_pcie_ep_deinit(&pcie_ep->pci.ep); return ret; } From 2cebf68a24abb0552ea59cf928829acd51f8b175 Mon Sep 17 00:00:00 2001 From: Manivannan Sadhasivam Date: Wed, 11 Sep 2024 20:56:26 +0530 Subject: [PATCH 204/302] PCI: dwc: Rename 'dw_pcie::link_gen' to 'dw_pcie::max_link_speed' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 'link_gen' field is now holding the maximum supported link speed set either by the controller driver or by DT through 'max-link-speed' property. However, the name 'link_gen' sounds like the negotiated link speed of the PCIe link. So rename it to 'max_link_speed' to make it clear that it holds the maximum supported link speed of the controller. Link: https://lore.kernel.org/linux-pci/20240911-pci-qcom-gen4-stability-v7-1-743f5c1fd027@linaro.org Tested-by: Johan Hovold Signed-off-by: Manivannan Sadhasivam [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Frank Li Reviewed-by: Johan Hovold --- drivers/pci/controller/dwc/pci-imx6.c | 8 ++++---- drivers/pci/controller/dwc/pcie-designware.c | 12 ++++++------ drivers/pci/controller/dwc/pcie-designware.h | 2 +- drivers/pci/controller/dwc/pcie-intel-gw.c | 4 ++-- drivers/pci/controller/dwc/pcie-rcar-gen4.c | 6 +++--- drivers/pci/controller/dwc/pcie-spear13xx.c | 2 +- 6 files changed, 17 insertions(+), 17 deletions(-) diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c index 964d67756eb2..ef12a4f31740 100644 --- a/drivers/pci/controller/dwc/pci-imx6.c +++ b/drivers/pci/controller/dwc/pci-imx6.c @@ -847,12 +847,12 @@ static int imx6_pcie_start_link(struct dw_pcie *pci) if (ret) goto err_reset_phy; - if (pci->link_gen > 1) { + if (pci->max_link_speed > 1) { /* Allow faster modes after the link is up */ dw_pcie_dbi_ro_wr_en(pci); tmp = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP); tmp &= ~PCI_EXP_LNKCAP_SLS; - tmp |= pci->link_gen; + tmp |= pci->max_link_speed; dw_pcie_writel_dbi(pci, offset + PCI_EXP_LNKCAP, tmp); /* @@ -1386,8 +1386,8 @@ static int imx6_pcie_probe(struct platform_device *pdev) imx6_pcie->tx_swing_low = 127; /* Limit link speed */ - pci->link_gen = 1; - of_property_read_u32(node, "fsl,max-link-speed", &pci->link_gen); + pci->max_link_speed = 1; + of_property_read_u32(node, "fsl,max-link-speed", &pci->max_link_speed); imx6_pcie->vpcie = devm_regulator_get_optional(&pdev->dev, "vpcie"); if (IS_ERR(imx6_pcie->vpcie)) { diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c index bc3a5d6b0177..95ae6c65b28f 100644 --- a/drivers/pci/controller/dwc/pcie-designware.c +++ b/drivers/pci/controller/dwc/pcie-designware.c @@ -168,8 +168,8 @@ int dw_pcie_get_resources(struct dw_pcie *pci) return ret; } - if (pci->link_gen < 1) - pci->link_gen = of_pci_get_max_link_speed(np); + if (pci->max_link_speed < 1) + pci->max_link_speed = of_pci_get_max_link_speed(np); of_property_read_u32(np, "num-lanes", &pci->num_lanes); @@ -689,7 +689,7 @@ void dw_pcie_upconfig_setup(struct dw_pcie *pci) } EXPORT_SYMBOL_GPL(dw_pcie_upconfig_setup); -static void dw_pcie_link_set_max_speed(struct dw_pcie *pci, u32 link_gen) +static void dw_pcie_link_set_max_speed(struct dw_pcie *pci, u32 max_link_speed) { u32 cap, ctrl2, link_speed; u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); @@ -698,7 +698,7 @@ static void dw_pcie_link_set_max_speed(struct dw_pcie *pci, u32 link_gen) ctrl2 = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCTL2); ctrl2 &= ~PCI_EXP_LNKCTL2_TLS; - switch (pcie_link_speed[link_gen]) { + switch (pcie_link_speed[max_link_speed]) { case PCIE_SPEED_2_5GT: link_speed = PCI_EXP_LNKCTL2_TLS_2_5GT; break; @@ -1060,8 +1060,8 @@ void dw_pcie_setup(struct dw_pcie *pci) { u32 val; - if (pci->link_gen > 0) - dw_pcie_link_set_max_speed(pci, pci->link_gen); + if (pci->max_link_speed > 0) + dw_pcie_link_set_max_speed(pci, pci->max_link_speed); /* Configure Gen1 N_FTS */ if (pci->n_fts[0]) { diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h index e518f81ea80c..1a1769244e83 100644 --- a/drivers/pci/controller/dwc/pcie-designware.h +++ b/drivers/pci/controller/dwc/pcie-designware.h @@ -423,7 +423,7 @@ struct dw_pcie { u32 type; unsigned long caps; int num_lanes; - int link_gen; + int max_link_speed; u8 n_fts[2]; struct dw_edma_chip edma; struct clk_bulk_data app_clks[DW_PCIE_NUM_APP_CLKS]; diff --git a/drivers/pci/controller/dwc/pcie-intel-gw.c b/drivers/pci/controller/dwc/pcie-intel-gw.c index acbe4f6d3291..676d2aba4fbd 100644 --- a/drivers/pci/controller/dwc/pcie-intel-gw.c +++ b/drivers/pci/controller/dwc/pcie-intel-gw.c @@ -132,7 +132,7 @@ static void intel_pcie_link_setup(struct intel_pcie *pcie) static void intel_pcie_init_n_fts(struct dw_pcie *pci) { - switch (pci->link_gen) { + switch (pci->max_link_speed) { case 3: pci->n_fts[1] = PORT_AFR_N_FTS_GEN3; break; @@ -252,7 +252,7 @@ static int intel_pcie_wait_l2(struct intel_pcie *pcie) int ret; struct dw_pcie *pci = &pcie->pci; - if (pci->link_gen < 3) + if (pci->max_link_speed < 3) return 0; /* Send PME_TURN_OFF message */ diff --git a/drivers/pci/controller/dwc/pcie-rcar-gen4.c b/drivers/pci/controller/dwc/pcie-rcar-gen4.c index f0f3ebd1a033..00ad4832f2cf 100644 --- a/drivers/pci/controller/dwc/pcie-rcar-gen4.c +++ b/drivers/pci/controller/dwc/pcie-rcar-gen4.c @@ -141,10 +141,10 @@ static int rcar_gen4_pcie_start_link(struct dw_pcie *dw) } /* - * Require direct speed change with retrying here if the link_gen is - * PCIe Gen2 or higher. + * Require direct speed change with retrying here if the max_link_speed + * is PCIe Gen2 or higher. */ - changes = min_not_zero(dw->link_gen, RCAR_MAX_LINK_SPEED) - 1; + changes = min_not_zero(dw->max_link_speed, RCAR_MAX_LINK_SPEED) - 1; /* * Since dw_pcie_setup_rc() sets it once, PCIe Gen2 will be trained. diff --git a/drivers/pci/controller/dwc/pcie-spear13xx.c b/drivers/pci/controller/dwc/pcie-spear13xx.c index 201dced209f0..ff986ced56b2 100644 --- a/drivers/pci/controller/dwc/pcie-spear13xx.c +++ b/drivers/pci/controller/dwc/pcie-spear13xx.c @@ -233,7 +233,7 @@ static int spear13xx_pcie_probe(struct platform_device *pdev) } if (of_property_read_bool(np, "st,pcie-is-gen1")) - pci->link_gen = 1; + pci->max_link_speed = 1; platform_set_drvdata(pdev, spear13xx_pcie); From 19a69cbd9d436fe503e5cb6dade76fe371244d4f Mon Sep 17 00:00:00 2001 From: Manivannan Sadhasivam Date: Wed, 11 Sep 2024 20:56:27 +0530 Subject: [PATCH 205/302] PCI: dwc: Always cache the maximum link speed value in dw_pcie::max_link_speed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently, the dw_pcie::max_link_speed has a valid value only if the controller driver restricts the maximum link speed in the driver or if the platform does so in the devicetree using the 'max-link-speed' property. But having the maximum supported link speed of the platform would be helpful for the vendor drivers to configure any link specific settings. So in the case of non-valid value in dw_pcie::max_link_speed, just cache the hardware default value from Link Capability register. While at it, remove the 'max_link_speed' argument to the dw_pcie_link_set_max_speed() function since the value can be retrieved within the function. Link: https://lore.kernel.org/linux-pci/20240911-pci-qcom-gen4-stability-v7-2-743f5c1fd027@linaro.org Tested-by: Johan Hovold Signed-off-by: Manivannan Sadhasivam [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Frank Li Reviewed-by: Johan Hovold --- drivers/pci/controller/dwc/pcie-designware.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c index 95ae6c65b28f..6d6cbc8b5b2c 100644 --- a/drivers/pci/controller/dwc/pcie-designware.c +++ b/drivers/pci/controller/dwc/pcie-designware.c @@ -689,16 +689,27 @@ void dw_pcie_upconfig_setup(struct dw_pcie *pci) } EXPORT_SYMBOL_GPL(dw_pcie_upconfig_setup); -static void dw_pcie_link_set_max_speed(struct dw_pcie *pci, u32 max_link_speed) +static void dw_pcie_link_set_max_speed(struct dw_pcie *pci) { u32 cap, ctrl2, link_speed; u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); cap = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP); + + /* + * Even if the platform doesn't want to limit the maximum link speed, + * just cache the hardware default value so that the vendor drivers can + * use it to do any link specific configuration. + */ + if (pci->max_link_speed < 1) { + pci->max_link_speed = FIELD_GET(PCI_EXP_LNKCAP_SLS, cap); + return; + } + ctrl2 = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCTL2); ctrl2 &= ~PCI_EXP_LNKCTL2_TLS; - switch (pcie_link_speed[max_link_speed]) { + switch (pcie_link_speed[pci->max_link_speed]) { case PCIE_SPEED_2_5GT: link_speed = PCI_EXP_LNKCTL2_TLS_2_5GT; break; @@ -1060,8 +1071,7 @@ void dw_pcie_setup(struct dw_pcie *pci) { u32 val; - if (pci->max_link_speed > 0) - dw_pcie_link_set_max_speed(pci, pci->max_link_speed); + dw_pcie_link_set_max_speed(pci); /* Configure Gen1 N_FTS */ if (pci->n_fts[0]) { From d45736b5984954da71292d858f277bac9c70cd2e Mon Sep 17 00:00:00 2001 From: Shashank Babu Chinta Venkata Date: Wed, 11 Sep 2024 20:56:28 +0530 Subject: [PATCH 206/302] PCI: qcom: Add equalization settings for 16.0 GT/s MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit During high data transmission rates such as 16.0 GT/s, there is an increased risk of signal loss due to poor channel quality and interference. This can impact receiver's ability to capture signals accurately. Hence, as signal compensation is achieved through appropriate lane equalization, apply lane equalization settings at both transmitter and receiver which results in an increase in the PCIe signal strength. While at it, modify the pcie-tegra194 driver to make use of the common GEN3_EQ_CONTROL_OFF definitions in pcie-designware.h. Link: https://lore.kernel.org/linux-pci/20240911-pci-qcom-gen4-stability-v7-3-743f5c1fd027@linaro.org Tested-by: Johan Hovold Signed-off-by: Shashank Babu Chinta Venkata [mani: dropped the code refactoring and minor changes] Signed-off-by: Manivannan Sadhasivam [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Johan Hovold Reviewed-by: Manivannan Sadhasivam --- MAINTAINERS | 4 +- drivers/pci/controller/dwc/Kconfig | 5 ++ drivers/pci/controller/dwc/Makefile | 1 + drivers/pci/controller/dwc/pcie-designware.h | 13 +++++ drivers/pci/controller/dwc/pcie-qcom-common.c | 47 +++++++++++++++++++ drivers/pci/controller/dwc/pcie-qcom-common.h | 13 +++++ drivers/pci/controller/dwc/pcie-qcom-ep.c | 4 ++ drivers/pci/controller/dwc/pcie-qcom.c | 4 ++ drivers/pci/controller/dwc/pcie-tegra194.c | 19 +++----- 9 files changed, 97 insertions(+), 13 deletions(-) create mode 100644 drivers/pci/controller/dwc/pcie-qcom-common.c create mode 100644 drivers/pci/controller/dwc/pcie-qcom-common.h diff --git a/MAINTAINERS b/MAINTAINERS index 42decde38320..90a88d1041da 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2728,7 +2728,7 @@ F: drivers/iommu/msm* F: drivers/mfd/ssbi.c F: drivers/mmc/host/mmci_qcom* F: drivers/mmc/host/sdhci-msm.c -F: drivers/pci/controller/dwc/pcie-qcom.c +F: drivers/pci/controller/dwc/pcie-qcom* F: drivers/phy/qualcomm/ F: drivers/power/*/msm* F: drivers/reset/reset-qcom-* @@ -17754,6 +17754,7 @@ M: Manivannan Sadhasivam L: linux-pci@vger.kernel.org L: linux-arm-msm@vger.kernel.org S: Maintained +F: drivers/pci/controller/dwc/pcie-qcom-common.c F: drivers/pci/controller/dwc/pcie-qcom.c PCIE DRIVER FOR ROCKCHIP @@ -17790,6 +17791,7 @@ L: linux-pci@vger.kernel.org L: linux-arm-msm@vger.kernel.org S: Maintained F: Documentation/devicetree/bindings/pci/qcom,pcie-ep.yaml +F: drivers/pci/controller/dwc/pcie-qcom-common.c F: drivers/pci/controller/dwc/pcie-qcom-ep.c PCMCIA SUBSYSTEM diff --git a/drivers/pci/controller/dwc/Kconfig b/drivers/pci/controller/dwc/Kconfig index 4c38181acffa..b6d6778b0698 100644 --- a/drivers/pci/controller/dwc/Kconfig +++ b/drivers/pci/controller/dwc/Kconfig @@ -265,12 +265,16 @@ config PCIE_DW_PLAT_EP order to enable device-specific features PCI_DW_PLAT_EP must be selected. +config PCIE_QCOM_COMMON + bool + config PCIE_QCOM bool "Qualcomm PCIe controller (host mode)" depends on OF && (ARCH_QCOM || COMPILE_TEST) depends on PCI_MSI select PCIE_DW_HOST select CRC8 + select PCIE_QCOM_COMMON help Say Y here to enable PCIe controller support on Qualcomm SoCs. The PCIe controller uses the DesignWare core plus Qualcomm-specific @@ -281,6 +285,7 @@ config PCIE_QCOM_EP depends on OF && (ARCH_QCOM || COMPILE_TEST) depends on PCI_ENDPOINT select PCIE_DW_EP + select PCIE_QCOM_COMMON help Say Y here to enable support for the PCIe controllers on Qualcomm SoCs to work in endpoint mode. The PCIe controller uses the DesignWare core diff --git a/drivers/pci/controller/dwc/Makefile b/drivers/pci/controller/dwc/Makefile index ec215b3d6191..a8308d9ea986 100644 --- a/drivers/pci/controller/dwc/Makefile +++ b/drivers/pci/controller/dwc/Makefile @@ -12,6 +12,7 @@ obj-$(CONFIG_PCIE_SPEAR13XX) += pcie-spear13xx.o obj-$(CONFIG_PCI_KEYSTONE) += pci-keystone.o obj-$(CONFIG_PCI_LAYERSCAPE) += pci-layerscape.o obj-$(CONFIG_PCI_LAYERSCAPE_EP) += pci-layerscape-ep.o +obj-$(CONFIG_PCIE_QCOM_COMMON) += pcie-qcom-common.o obj-$(CONFIG_PCIE_QCOM) += pcie-qcom.o obj-$(CONFIG_PCIE_QCOM_EP) += pcie-qcom-ep.o obj-$(CONFIG_PCIE_ARMADA_8K) += pcie-armada8k.o diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h index 1a1769244e83..67cc2677ab2e 100644 --- a/drivers/pci/controller/dwc/pcie-designware.h +++ b/drivers/pci/controller/dwc/pcie-designware.h @@ -125,6 +125,19 @@ #define GEN3_RELATED_OFF_GEN3_EQ_DISABLE BIT(16) #define GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT 24 #define GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK GENMASK(25, 24) +#define GEN3_RELATED_OFF_RATE_SHADOW_SEL_16_0GT 0x1 + +#define GEN3_EQ_CONTROL_OFF 0x8A8 +#define GEN3_EQ_CONTROL_OFF_FB_MODE GENMASK(3, 0) +#define GEN3_EQ_CONTROL_OFF_PHASE23_EXIT_MODE BIT(4) +#define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC GENMASK(23, 8) +#define GEN3_EQ_CONTROL_OFF_FOM_INC_INITIAL_EVAL BIT(24) + +#define GEN3_EQ_FB_MODE_DIR_CHANGE_OFF 0x8AC +#define GEN3_EQ_FMDC_T_MIN_PHASE23 GENMASK(4, 0) +#define GEN3_EQ_FMDC_N_EVALS GENMASK(9, 5) +#define GEN3_EQ_FMDC_MAX_PRE_CUSROR_DELTA GENMASK(13, 10) +#define GEN3_EQ_FMDC_MAX_POST_CUSROR_DELTA GENMASK(17, 14) #define PCIE_PORT_MULTI_LANE_CTRL 0x8C0 #define PORT_MLTI_UPCFG_SUPPORT BIT(7) diff --git a/drivers/pci/controller/dwc/pcie-qcom-common.c b/drivers/pci/controller/dwc/pcie-qcom-common.c new file mode 100644 index 000000000000..596a35449de1 --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-qcom-common.c @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include + +#include "pcie-designware.h" +#include "pcie-qcom-common.h" + +void qcom_pcie_common_set_16gt_equalization(struct dw_pcie *pci) +{ + u32 reg; + + /* + * GEN3_RELATED_OFF register is repurposed to apply equalization + * settings at various data transmission rates through registers namely + * GEN3_EQ_*. The RATE_SHADOW_SEL bit field of GEN3_RELATED_OFF + * determines the data rate for which these equalization settings are + * applied. + */ + reg = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); + reg &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL; + reg &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK; + reg |= FIELD_PREP(GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK, + GEN3_RELATED_OFF_RATE_SHADOW_SEL_16_0GT); + dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, reg); + + reg = dw_pcie_readl_dbi(pci, GEN3_EQ_FB_MODE_DIR_CHANGE_OFF); + reg &= ~(GEN3_EQ_FMDC_T_MIN_PHASE23 | + GEN3_EQ_FMDC_N_EVALS | + GEN3_EQ_FMDC_MAX_PRE_CUSROR_DELTA | + GEN3_EQ_FMDC_MAX_POST_CUSROR_DELTA); + reg |= FIELD_PREP(GEN3_EQ_FMDC_T_MIN_PHASE23, 0x1) | + FIELD_PREP(GEN3_EQ_FMDC_N_EVALS, 0xd) | + FIELD_PREP(GEN3_EQ_FMDC_MAX_PRE_CUSROR_DELTA, 0x5) | + FIELD_PREP(GEN3_EQ_FMDC_MAX_POST_CUSROR_DELTA, 0x5); + dw_pcie_writel_dbi(pci, GEN3_EQ_FB_MODE_DIR_CHANGE_OFF, reg); + + reg = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF); + reg &= ~(GEN3_EQ_CONTROL_OFF_FB_MODE | + GEN3_EQ_CONTROL_OFF_PHASE23_EXIT_MODE | + GEN3_EQ_CONTROL_OFF_FOM_INC_INITIAL_EVAL | + GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC); + dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, reg); +} +EXPORT_SYMBOL_GPL(qcom_pcie_common_set_16gt_equalization); diff --git a/drivers/pci/controller/dwc/pcie-qcom-common.h b/drivers/pci/controller/dwc/pcie-qcom-common.h new file mode 100644 index 000000000000..536387e02e29 --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-qcom-common.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef _PCIE_QCOM_COMMON_H +#define _PCIE_QCOM_COMMON_H + +struct dw_pcie; + +void qcom_pcie_common_set_16gt_equalization(struct dw_pcie *pci); + +#endif diff --git a/drivers/pci/controller/dwc/pcie-qcom-ep.c b/drivers/pci/controller/dwc/pcie-qcom-ep.c index 3c1b695638f3..310d52393392 100644 --- a/drivers/pci/controller/dwc/pcie-qcom-ep.c +++ b/drivers/pci/controller/dwc/pcie-qcom-ep.c @@ -25,6 +25,7 @@ #include "../../pci.h" #include "pcie-designware.h" +#include "pcie-qcom-common.h" /* PARF registers */ #define PARF_SYS_CTRL 0x00 @@ -486,6 +487,9 @@ static int qcom_pcie_perst_deassert(struct dw_pcie *pci) goto err_disable_resources; } + if (pcie_link_speed[pci->max_link_speed] == PCIE_SPEED_16_0GT) + qcom_pcie_common_set_16gt_equalization(pci); + /* * The physical address of the MMIO region which is exposed as the BAR * should be written to MHI BASE registers. diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c index 1923266acea8..1e84366ac0bb 100644 --- a/drivers/pci/controller/dwc/pcie-qcom.c +++ b/drivers/pci/controller/dwc/pcie-qcom.c @@ -35,6 +35,7 @@ #include "../../pci.h" #include "pcie-designware.h" +#include "pcie-qcom-common.h" /* PARF registers */ #define PARF_SYS_CTRL 0x00 @@ -295,6 +296,9 @@ static int qcom_pcie_start_link(struct dw_pcie *pci) { struct qcom_pcie *pcie = to_qcom_pcie(pci); + if (pcie_link_speed[pci->max_link_speed] == PCIE_SPEED_16_0GT) + qcom_pcie_common_set_16gt_equalization(pci); + /* Enable Link Training state machine */ if (pcie->cfg->ops->ltssm_enable) pcie->cfg->ops->ltssm_enable(pcie); diff --git a/drivers/pci/controller/dwc/pcie-tegra194.c b/drivers/pci/controller/dwc/pcie-tegra194.c index 4bf7b433417a..6e03e14151d6 100644 --- a/drivers/pci/controller/dwc/pcie-tegra194.c +++ b/drivers/pci/controller/dwc/pcie-tegra194.c @@ -177,11 +177,6 @@ #define N_FTS_VAL 52 #define FTS_VAL 52 -#define GEN3_EQ_CONTROL_OFF 0x8a8 -#define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT 8 -#define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK GENMASK(23, 8) -#define GEN3_EQ_CONTROL_OFF_FB_MODE_MASK GENMASK(3, 0) - #define PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT 0x8D0 #define AMBA_ERROR_RESPONSE_CRS_SHIFT 3 #define AMBA_ERROR_RESPONSE_CRS_MASK GENMASK(1, 0) @@ -861,9 +856,9 @@ static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie) dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF); - val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK; - val |= (0x3ff << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT); - val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK; + val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC; + val |= FIELD_PREP(GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC, 0x3ff); + val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE; dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val); val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); @@ -872,10 +867,10 @@ static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie) dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF); - val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK; - val |= (pcie->of_data->gen4_preset_vec << - GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT); - val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK; + val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC; + val |= FIELD_PREP(GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC, + pcie->of_data->gen4_preset_vec); + val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE; dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val); val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); From d14bc28af34fb8b599c1cc4ce24a2833e60ade8f Mon Sep 17 00:00:00 2001 From: Shashank Babu Chinta Venkata Date: Wed, 11 Sep 2024 20:56:29 +0530 Subject: [PATCH 207/302] PCI: qcom: Add RX lane margining settings for 16.0 GT/s MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add RX lane margining settings for 16.0 GT/s (GEN 4) data rate. These settings improve link stability while operating at high date rates and helps to improve signal quality. Link: https://lore.kernel.org/linux-pci/20240911-pci-qcom-gen4-stability-v7-4-743f5c1fd027@linaro.org Tested-by: Johan Hovold Signed-off-by: Shashank Babu Chinta Venkata [mani: dropped the code refactoring and minor changes] Signed-off-by: Manivannan Sadhasivam [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Johan Hovold Reviewed-by: Manivannan Sadhasivam --- drivers/pci/controller/dwc/pcie-designware.h | 18 +++++++++++ drivers/pci/controller/dwc/pcie-qcom-common.c | 31 +++++++++++++++++++ drivers/pci/controller/dwc/pcie-qcom-common.h | 1 + drivers/pci/controller/dwc/pcie-qcom-ep.c | 4 ++- drivers/pci/controller/dwc/pcie-qcom.c | 4 ++- 5 files changed, 56 insertions(+), 2 deletions(-) diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h index 67cc2677ab2e..347ab74ac35a 100644 --- a/drivers/pci/controller/dwc/pcie-designware.h +++ b/drivers/pci/controller/dwc/pcie-designware.h @@ -210,6 +210,24 @@ #define PCIE_PL_CHK_REG_ERR_ADDR 0xB28 +/* + * 16.0 GT/s (Gen 4) lane margining register definitions + */ +#define GEN4_LANE_MARGINING_1_OFF 0xB80 +#define MARGINING_MAX_VOLTAGE_OFFSET GENMASK(29, 24) +#define MARGINING_NUM_VOLTAGE_STEPS GENMASK(22, 16) +#define MARGINING_MAX_TIMING_OFFSET GENMASK(13, 8) +#define MARGINING_NUM_TIMING_STEPS GENMASK(5, 0) + +#define GEN4_LANE_MARGINING_2_OFF 0xB84 +#define MARGINING_IND_ERROR_SAMPLER BIT(28) +#define MARGINING_SAMPLE_REPORTING_METHOD BIT(27) +#define MARGINING_IND_LEFT_RIGHT_TIMING BIT(26) +#define MARGINING_IND_UP_DOWN_VOLTAGE BIT(25) +#define MARGINING_VOLTAGE_SUPPORTED BIT(24) +#define MARGINING_MAXLANES GENMASK(20, 16) +#define MARGINING_SAMPLE_RATE_TIMING GENMASK(13, 8) +#define MARGINING_SAMPLE_RATE_VOLTAGE GENMASK(5, 0) /* * iATU Unroll-specific register definitions * From 4.80 core version the address translation will be made by unroll diff --git a/drivers/pci/controller/dwc/pcie-qcom-common.c b/drivers/pci/controller/dwc/pcie-qcom-common.c index 596a35449de1..3aad19b56da8 100644 --- a/drivers/pci/controller/dwc/pcie-qcom-common.c +++ b/drivers/pci/controller/dwc/pcie-qcom-common.c @@ -45,3 +45,34 @@ void qcom_pcie_common_set_16gt_equalization(struct dw_pcie *pci) dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, reg); } EXPORT_SYMBOL_GPL(qcom_pcie_common_set_16gt_equalization); + +void qcom_pcie_common_set_16gt_lane_margining(struct dw_pcie *pci) +{ + u32 reg; + + reg = dw_pcie_readl_dbi(pci, GEN4_LANE_MARGINING_1_OFF); + reg &= ~(MARGINING_MAX_VOLTAGE_OFFSET | + MARGINING_NUM_VOLTAGE_STEPS | + MARGINING_MAX_TIMING_OFFSET | + MARGINING_NUM_TIMING_STEPS); + reg |= FIELD_PREP(MARGINING_MAX_VOLTAGE_OFFSET, 0x24) | + FIELD_PREP(MARGINING_NUM_VOLTAGE_STEPS, 0x78) | + FIELD_PREP(MARGINING_MAX_TIMING_OFFSET, 0x32) | + FIELD_PREP(MARGINING_NUM_TIMING_STEPS, 0x10); + dw_pcie_writel_dbi(pci, GEN4_LANE_MARGINING_1_OFF, reg); + + reg = dw_pcie_readl_dbi(pci, GEN4_LANE_MARGINING_2_OFF); + reg |= MARGINING_IND_ERROR_SAMPLER | + MARGINING_SAMPLE_REPORTING_METHOD | + MARGINING_IND_LEFT_RIGHT_TIMING | + MARGINING_VOLTAGE_SUPPORTED; + reg &= ~(MARGINING_IND_UP_DOWN_VOLTAGE | + MARGINING_MAXLANES | + MARGINING_SAMPLE_RATE_TIMING | + MARGINING_SAMPLE_RATE_VOLTAGE); + reg |= FIELD_PREP(MARGINING_MAXLANES, pci->num_lanes) | + FIELD_PREP(MARGINING_SAMPLE_RATE_TIMING, 0x3f) | + FIELD_PREP(MARGINING_SAMPLE_RATE_VOLTAGE, 0x3f); + dw_pcie_writel_dbi(pci, GEN4_LANE_MARGINING_2_OFF, reg); +} +EXPORT_SYMBOL_GPL(qcom_pcie_common_set_16gt_lane_margining); diff --git a/drivers/pci/controller/dwc/pcie-qcom-common.h b/drivers/pci/controller/dwc/pcie-qcom-common.h index 536387e02e29..7d88d29e4766 100644 --- a/drivers/pci/controller/dwc/pcie-qcom-common.h +++ b/drivers/pci/controller/dwc/pcie-qcom-common.h @@ -9,5 +9,6 @@ struct dw_pcie; void qcom_pcie_common_set_16gt_equalization(struct dw_pcie *pci); +void qcom_pcie_common_set_16gt_lane_margining(struct dw_pcie *pci); #endif diff --git a/drivers/pci/controller/dwc/pcie-qcom-ep.c b/drivers/pci/controller/dwc/pcie-qcom-ep.c index 310d52393392..aa8443c995e7 100644 --- a/drivers/pci/controller/dwc/pcie-qcom-ep.c +++ b/drivers/pci/controller/dwc/pcie-qcom-ep.c @@ -487,8 +487,10 @@ static int qcom_pcie_perst_deassert(struct dw_pcie *pci) goto err_disable_resources; } - if (pcie_link_speed[pci->max_link_speed] == PCIE_SPEED_16_0GT) + if (pcie_link_speed[pci->max_link_speed] == PCIE_SPEED_16_0GT) { qcom_pcie_common_set_16gt_equalization(pci); + qcom_pcie_common_set_16gt_lane_margining(pci); + } /* * The physical address of the MMIO region which is exposed as the BAR diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c index 1e84366ac0bb..22028d10b994 100644 --- a/drivers/pci/controller/dwc/pcie-qcom.c +++ b/drivers/pci/controller/dwc/pcie-qcom.c @@ -296,8 +296,10 @@ static int qcom_pcie_start_link(struct dw_pcie *pci) { struct qcom_pcie *pcie = to_qcom_pcie(pci); - if (pcie_link_speed[pci->max_link_speed] == PCIE_SPEED_16_0GT) + if (pcie_link_speed[pci->max_link_speed] == PCIE_SPEED_16_0GT) { qcom_pcie_common_set_16gt_equalization(pci); + qcom_pcie_common_set_16gt_lane_margining(pci); + } /* Enable Link Training state machine */ if (pcie->cfg->ops->ltssm_enable) From 03f84b3baba7836bdfc162c19288d5ce1aa92890 Mon Sep 17 00:00:00 2001 From: Siddharth Vadapalli Date: Tue, 27 Aug 2024 17:54:21 +0530 Subject: [PATCH 208/302] PCI: dra7xx: Fix threaded IRQ request for "dra7xx-pcie-main" IRQ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit da87d35a6e51 ("PCI: dra7xx: Use threaded IRQ handler for "dra7xx-pcie-main" IRQ") switched from devm_request_irq() to devm_request_threaded_irq() for the "dra7xx-pcie-main" interrupt. Since the primary handler was set to NULL, the "IRQF_ONESHOT" flag should have also been set. Fix this. Fixes: da87d35a6e51 ("PCI: dra7xx: Use threaded IRQ handler for "dra7xx-pcie-main" IRQ") Suggested-by: Vignesh Raghavendra Link: https://lore.kernel.org/linux-pci/20240827122422.985547-2-s-vadapalli@ti.com Reported-by: Udit Kumar Signed-off-by: Siddharth Vadapalli Signed-off-by: Krzysztof Wilczyński Reviewed-by: Kevin Hilman Reviewed-by: Manivannan Sadhasivam Cc: stable@vger.kernel.org --- drivers/pci/controller/dwc/pci-dra7xx.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/pci/controller/dwc/pci-dra7xx.c b/drivers/pci/controller/dwc/pci-dra7xx.c index 4fe3b0cb72ec..20fb50741f3d 100644 --- a/drivers/pci/controller/dwc/pci-dra7xx.c +++ b/drivers/pci/controller/dwc/pci-dra7xx.c @@ -850,7 +850,8 @@ static int dra7xx_pcie_probe(struct platform_device *pdev) dra7xx->mode = mode; ret = devm_request_threaded_irq(dev, irq, NULL, dra7xx_pcie_irq_handler, - IRQF_SHARED, "dra7xx-pcie-main", dra7xx); + IRQF_SHARED | IRQF_ONESHOT, + "dra7xx-pcie-main", dra7xx); if (ret) { dev_err(dev, "failed to request irq\n"); goto err_gpio; From 4d60f6d4b8fa4d7bad4aeb2b3ee5c10425bc60a4 Mon Sep 17 00:00:00 2001 From: Siddharth Vadapalli Date: Tue, 27 Aug 2024 17:54:22 +0530 Subject: [PATCH 209/302] PCI: dra7xx: Fix error handling when IRQ request fails in probe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit d4c7d1a089d6 ("PCI: dwc: dra7xx: Push request_irq() call to the bottom of probe") moved the IRQ request for "dra7xx-pcie-main" towards the end of dra7xx_pcie_probe(). However, the error handling does not take into account the initialization performed by either dra7xx_add_pcie_port() or dra7xx_add_pcie_ep(), depending on the mode of operation. Fix the error handling to address this. Fixes: d4c7d1a089d6 ("PCI: dwc: dra7xx: Push request_irq() call to the bottom of probe") Link: https://lore.kernel.org/linux-pci/20240827122422.985547-3-s-vadapalli@ti.com Tested-by: Udit Kumar Signed-off-by: Siddharth Vadapalli [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński Reviewed-by: Kevin Hilman Reviewed-by: Manivannan Sadhasivam Cc: stable@vger.kernel.org --- drivers/pci/controller/dwc/pci-dra7xx.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/pci/controller/dwc/pci-dra7xx.c b/drivers/pci/controller/dwc/pci-dra7xx.c index 20fb50741f3d..5c62e1a3ba52 100644 --- a/drivers/pci/controller/dwc/pci-dra7xx.c +++ b/drivers/pci/controller/dwc/pci-dra7xx.c @@ -854,11 +854,17 @@ static int dra7xx_pcie_probe(struct platform_device *pdev) "dra7xx-pcie-main", dra7xx); if (ret) { dev_err(dev, "failed to request irq\n"); - goto err_gpio; + goto err_deinit; } return 0; +err_deinit: + if (dra7xx->mode == DW_PCIE_RC_TYPE) + dw_pcie_host_deinit(&dra7xx->pci->pp); + else + dw_pcie_ep_deinit(&dra7xx->pci->ep); + err_gpio: err_get_sync: pm_runtime_put(dev); From 0cca961a026177af69044f10d6ae76d8ce043764 Mon Sep 17 00:00:00 2001 From: Manivannan Sadhasivam Date: Thu, 12 Sep 2024 11:00:25 +0530 Subject: [PATCH 210/302] PCI: Pass domain number to pci_bus_release_domain_nr() explicitly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pci_bus_release_domain_nr() API is supposed to free the domain number allocated by pci_bus_find_domain_nr(). Most of the callers of pci_bus_find_domain_nr(), store the domain number in pci_bus::domain_nr. As such, the pci_bus_release_domain_nr() implicitly frees the domain number by dereferencing 'struct pci_bus'. However, one of the callers of this API, the PCI endpoint subsystem, doesn't have 'struct pci_bus', so it only passes NULL. Due to this, the API will end up dereferencing the NULL pointer. To fix this issue, pass the domain number to this API explicitly. Since 'struct pci_bus' is not used for anything else other than extracting the domain number, it makes sense to pass the domain number directly. Fixes: 0328947c5032 ("PCI: endpoint: Assign PCI domain number for endpoint controllers") Closes: https://lore.kernel.org/linux-pci/c0c40ddb-bf64-4b22-9dd1-8dbb18aa2813@stanley.mountain Link: https://lore.kernel.org/linux-pci/20240912053025.25314-1-manivannan.sadhasivam@linaro.org Reported-by: Dan Carpenter Signed-off-by: Manivannan Sadhasivam [kwilczynski: commit log] Signed-off-by: Krzysztof Wilczyński --- drivers/pci/endpoint/pci-epc-core.c | 2 +- drivers/pci/pci.c | 14 +++++++------- drivers/pci/probe.c | 2 +- drivers/pci/remove.c | 2 +- include/linux/pci.h | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/drivers/pci/endpoint/pci-epc-core.c b/drivers/pci/endpoint/pci-epc-core.c index 085a2de8b923..17f007109255 100644 --- a/drivers/pci/endpoint/pci-epc-core.c +++ b/drivers/pci/endpoint/pci-epc-core.c @@ -840,7 +840,7 @@ void pci_epc_destroy(struct pci_epc *epc) device_unregister(&epc->dev); #ifdef CONFIG_PCI_DOMAINS_GENERIC - pci_bus_release_domain_nr(NULL, &epc->dev); + pci_bus_release_domain_nr(&epc->dev, epc->domain_nr); #endif } EXPORT_SYMBOL_GPL(pci_epc_destroy); diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index e3a49f66982d..b86693c91753 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -6801,16 +6801,16 @@ static int of_pci_bus_find_domain_nr(struct device *parent) return ida_alloc(&pci_domain_nr_dynamic_ida, GFP_KERNEL); } -static void of_pci_bus_release_domain_nr(struct pci_bus *bus, struct device *parent) +static void of_pci_bus_release_domain_nr(struct device *parent, int domain_nr) { - if (bus->domain_nr < 0) + if (domain_nr < 0) return; /* Release domain from IDA where it was allocated. */ - if (of_get_pci_domain_nr(parent->of_node) == bus->domain_nr) - ida_free(&pci_domain_nr_static_ida, bus->domain_nr); + if (of_get_pci_domain_nr(parent->of_node) == domain_nr) + ida_free(&pci_domain_nr_static_ida, domain_nr); else - ida_free(&pci_domain_nr_dynamic_ida, bus->domain_nr); + ida_free(&pci_domain_nr_dynamic_ida, domain_nr); } int pci_bus_find_domain_nr(struct pci_bus *bus, struct device *parent) @@ -6819,11 +6819,11 @@ int pci_bus_find_domain_nr(struct pci_bus *bus, struct device *parent) acpi_pci_bus_find_domain_nr(bus); } -void pci_bus_release_domain_nr(struct pci_bus *bus, struct device *parent) +void pci_bus_release_domain_nr(struct device *parent, int domain_nr) { if (!acpi_disabled) return; - of_pci_bus_release_domain_nr(bus, parent); + of_pci_bus_release_domain_nr(parent, domain_nr); } #endif diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index b14b9876c030..e9e56bbb3b59 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -1061,7 +1061,7 @@ unregister: free: #ifdef CONFIG_PCI_DOMAINS_GENERIC - pci_bus_release_domain_nr(bus, parent); + pci_bus_release_domain_nr(parent, bus->domain_nr); #endif kfree(bus); return err; diff --git a/drivers/pci/remove.c b/drivers/pci/remove.c index 910387e5bdbf..d2523fdf9bae 100644 --- a/drivers/pci/remove.c +++ b/drivers/pci/remove.c @@ -163,7 +163,7 @@ void pci_remove_root_bus(struct pci_bus *bus) #ifdef CONFIG_PCI_DOMAINS_GENERIC /* Release domain_nr if it was dynamically allocated */ if (host_bridge->domain_nr == PCI_DOMAIN_NR_NOT_SET) - pci_bus_release_domain_nr(bus, host_bridge->dev.parent); + pci_bus_release_domain_nr(host_bridge->dev.parent, bus->domain_nr); #endif pci_remove_bus(bus); diff --git a/include/linux/pci.h b/include/linux/pci.h index 4cf89a4b4cbc..37d97bef060f 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -1884,7 +1884,7 @@ static inline int acpi_pci_bus_find_domain_nr(struct pci_bus *bus) { return 0; } #endif int pci_bus_find_domain_nr(struct pci_bus *bus, struct device *parent); -void pci_bus_release_domain_nr(struct pci_bus *bus, struct device *parent); +void pci_bus_release_domain_nr(struct device *parent, int domain_nr); #endif /* Some architectures require additional setup to direct VGA traffic */ From 5dd15cce0c5426fff7b30c792d908e93646cd8c5 Mon Sep 17 00:00:00 2001 From: zhang jiao Date: Mon, 2 Sep 2024 12:12:40 +0800 Subject: [PATCH 211/302] tools: PCI: Remove .*.cmd files with make clean MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove any leftover .*.cmd files with make clean. No functional changes intended. Link: https://lore.kernel.org/linux-pci/20240902041240.5475-1-zhangjiao2@cmss.chinamobile.com Signed-off-by: zhang jiao [kwilczynski: commit log, move .*.cmd before .*.d to align with other Makefiles, don't remove the newline] Signed-off-by: Krzysztof Wilczyński --- tools/pci/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pci/Makefile b/tools/pci/Makefile index 57744778b518..62d41f1a1e2c 100644 --- a/tools/pci/Makefile +++ b/tools/pci/Makefile @@ -42,7 +42,7 @@ $(OUTPUT)pcitest: $(PCITEST_IN) clean: rm -f $(ALL_PROGRAMS) rm -rf $(OUTPUT)include/ - find $(or $(OUTPUT),.) -name '*.o' -delete -o -name '\.*.d' -delete + find $(or $(OUTPUT),.) -name '*.o' -delete -o -name '\.*.cmd' -delete -o -name '\.*.d' -delete install: $(ALL_PROGRAMS) install -d -m 755 $(DESTDIR)$(bindir); \ From d0aac667f2e02f114bc6c3bf8c085a6060599aef Mon Sep 17 00:00:00 2001 From: zhang jiao Date: Wed, 11 Sep 2024 14:04:01 +0800 Subject: [PATCH 212/302] tools: PCI: Remove unused BILLION macro MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The macro BILLION is never referenced in the code. Remove it. Link: https://lore.kernel.org/r/20240911060401.9230-1-zhangjiao2@cmss.chinamobile.com Signed-off-by: zhang jiao Signed-off-by: Bjorn Helgaas Signed-off-by: Krzysztof Wilczyński --- tools/pci/pcitest.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/pci/pcitest.c b/tools/pci/pcitest.c index 441b54234635..470258009ddc 100644 --- a/tools/pci/pcitest.c +++ b/tools/pci/pcitest.c @@ -16,8 +16,6 @@ #include -#define BILLION 1E9 - static char *result[] = { "NOT OKAY", "OKAY" }; static char *irq[] = { "LEGACY", "MSI", "MSI-X" }; From 3cd3b49989fef8556b018fb8038581195b253b8d Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Tue, 23 Jul 2024 17:13:28 +0200 Subject: [PATCH 213/302] dt-bindings: PCI: qcom: Allow 'vddpe-3v3-supply' again Commit 756485bfbb85 ("dt-bindings: PCI: qcom,pcie-sc7280: Move SC7280 to dedicated schema") incorrectly removed 'vddpe-3v3-supply' from the bindings, which results in DT checker warnings like: arch/arm64/boot/dts/qcom/msm8996-sony-xperia-tone-dora.dtb: pcie@600000: Unevaluated properties are not allowed ('vddpe-3v3-supply' was unexpected) from schema $id: http://devicetree.org/schemas/pci/qcom,pcie.yaml# Note that this property has been part of the Qualcomm PCIe bindings since 2018 and would need to be deprecated rather than simply removed if there is a desire to replace it with 'vpcie3v3' which is used for some non-Qualcomm controllers. Link: https://lore.kernel.org/lkml/Zp_LPixNnh-2Fy5N@hovoldconsulting.com/ Fixes: 756485bfbb85 ("dt-bindings: PCI: qcom,pcie-sc7280: Move SC7280 to dedicated schema") Link: https://lore.kernel.org/r/20240723151328.684-1-johan+linaro@kernel.org Signed-off-by: Johan Hovold Signed-off-by: Bjorn Helgaas Reviewed-by: Krzysztof Kozlowski Reviewed-by: Manivannan Sadhasivam --- Documentation/devicetree/bindings/pci/qcom,pcie-common.yaml | 3 +++ Documentation/devicetree/bindings/pci/qcom,pcie-sc7280.yaml | 3 --- Documentation/devicetree/bindings/pci/qcom,pcie-sc8280xp.yaml | 3 --- Documentation/devicetree/bindings/pci/qcom,pcie.yaml | 3 +++ 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie-common.yaml b/Documentation/devicetree/bindings/pci/qcom,pcie-common.yaml index 0a39bbfcb28b..2b6f5a171f20 100644 --- a/Documentation/devicetree/bindings/pci/qcom,pcie-common.yaml +++ b/Documentation/devicetree/bindings/pci/qcom,pcie-common.yaml @@ -78,6 +78,9 @@ properties: description: GPIO controlled connection to WAKE# signal maxItems: 1 + vddpe-3v3-supply: + description: PCIe endpoint power supply + required: - reg - reg-names diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie-sc7280.yaml b/Documentation/devicetree/bindings/pci/qcom,pcie-sc7280.yaml index 5cf1f9165301..76cb9fbfd476 100644 --- a/Documentation/devicetree/bindings/pci/qcom,pcie-sc7280.yaml +++ b/Documentation/devicetree/bindings/pci/qcom,pcie-sc7280.yaml @@ -74,9 +74,6 @@ properties: items: - const: pci - vddpe-3v3-supply: - description: PCIe endpoint power supply - allOf: - $ref: qcom,pcie-common.yaml# diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie-sc8280xp.yaml b/Documentation/devicetree/bindings/pci/qcom,pcie-sc8280xp.yaml index 25c9f13ae977..15ba2385eb73 100644 --- a/Documentation/devicetree/bindings/pci/qcom,pcie-sc8280xp.yaml +++ b/Documentation/devicetree/bindings/pci/qcom,pcie-sc8280xp.yaml @@ -58,9 +58,6 @@ properties: items: - const: pci - vddpe-3v3-supply: - description: A phandle to the PCIe endpoint power supply - required: - interconnects - interconnect-names diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie.yaml b/Documentation/devicetree/bindings/pci/qcom,pcie.yaml index f867746b1ae5..ffabbac57fc1 100644 --- a/Documentation/devicetree/bindings/pci/qcom,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/qcom,pcie.yaml @@ -91,6 +91,9 @@ properties: vdda_refclk-supply: description: A phandle to the core analog power supply for IC which generates reference clock + vddpe-3v3-supply: + description: A phandle to the PCIe endpoint power supply + phys: maxItems: 1 From 7ee85f5515e86a4e2a2f51969795920733912bad Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Tue, 3 Sep 2024 10:55:36 +0100 Subject: [PATCH 214/302] btrfs: fix race setting file private on concurrent lseek using same fd When doing concurrent lseek(2) system calls against the same file descriptor, using multiple threads belonging to the same process, we have a short time window where a race happens and can result in a memory leak. The race happens like this: 1) A program opens a file descriptor for a file and then spawns two threads (with the pthreads library for example), lets call them task A and task B; 2) Task A calls lseek with SEEK_DATA or SEEK_HOLE and ends up at file.c:find_desired_extent() while holding a read lock on the inode; 3) At the start of find_desired_extent(), it extracts the file's private_data pointer into a local variable named 'private', which has a value of NULL; 4) Task B also calls lseek with SEEK_DATA or SEEK_HOLE, locks the inode in shared mode and enters file.c:find_desired_extent(), where it also extracts file->private_data into its local variable 'private', which has a NULL value; 5) Because it saw a NULL file private, task A allocates a private structure and assigns to the file structure; 6) Task B also saw a NULL file private so it also allocates its own file private and then assigns it to the same file structure, since both tasks are using the same file descriptor. At this point we leak the private structure allocated by task A. Besides the memory leak, there's also the detail that both tasks end up using the same cached state record in the private structure (struct btrfs_file_private::llseek_cached_state), which can result in a use-after-free problem since one task can free it while the other is still using it (only one task took a reference count on it). Also, sharing the cached state is not a good idea since it could result in incorrect results in the future - right now it should not be a problem because it end ups being used only in extent-io-tree.c:count_range_bits() where we do range validation before using the cached state. Fix this by protecting the private assignment and check of a file while holding the inode's spinlock and keep track of the task that allocated the private, so that it's used only by that task in order to prevent user-after-free issues with the cached state record as well as potentially using it incorrectly in the future. Fixes: 3c32c7212f16 ("btrfs: use cached state when looking for delalloc ranges with lseek") CC: stable@vger.kernel.org # 6.6+ Reviewed-by: Josef Bacik Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/btrfs_inode.h | 1 + fs/btrfs/ctree.h | 2 ++ fs/btrfs/file.c | 34 +++++++++++++++++++++++++++++++--- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/fs/btrfs/btrfs_inode.h b/fs/btrfs/btrfs_inode.h index 9a4b7c119318..e152fde888fc 100644 --- a/fs/btrfs/btrfs_inode.h +++ b/fs/btrfs/btrfs_inode.h @@ -152,6 +152,7 @@ struct btrfs_inode { * logged_trans), to access/update delalloc_bytes, new_delalloc_bytes, * defrag_bytes, disk_i_size, outstanding_extents, csum_bytes and to * update the VFS' inode number of bytes used. + * Also protects setting struct file::private_data. */ spinlock_t lock; diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 1a44fb9845e3..317a3712270f 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -463,6 +463,8 @@ struct btrfs_file_private { void *filldir_buf; u64 last_index; struct extent_state *llseek_cached_state; + /* Task that allocated this structure. */ + struct task_struct *owner_task; }; static inline u32 BTRFS_LEAF_DATA_SIZE(const struct btrfs_fs_info *info) diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index c5e36f58eb07..4fb521d91b06 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -3485,7 +3485,7 @@ static bool find_desired_extent_in_hole(struct btrfs_inode *inode, int whence, static loff_t find_desired_extent(struct file *file, loff_t offset, int whence) { struct btrfs_inode *inode = BTRFS_I(file->f_mapping->host); - struct btrfs_file_private *private = file->private_data; + struct btrfs_file_private *private; struct btrfs_fs_info *fs_info = inode->root->fs_info; struct extent_state *cached_state = NULL; struct extent_state **delalloc_cached_state; @@ -3513,7 +3513,19 @@ static loff_t find_desired_extent(struct file *file, loff_t offset, int whence) inode_get_bytes(&inode->vfs_inode) == i_size) return i_size; - if (!private) { + spin_lock(&inode->lock); + private = file->private_data; + spin_unlock(&inode->lock); + + if (private && private->owner_task != current) { + /* + * Not allocated by us, don't use it as its cached state is used + * by the task that allocated it and we don't want neither to + * mess with it nor get incorrect results because it reflects an + * invalid state for the current task. + */ + private = NULL; + } else if (!private) { private = kzalloc(sizeof(*private), GFP_KERNEL); /* * No worries if memory allocation failed. @@ -3521,7 +3533,23 @@ static loff_t find_desired_extent(struct file *file, loff_t offset, int whence) * lseek SEEK_HOLE/DATA calls to a file when there's delalloc, * so everything will still be correct. */ - file->private_data = private; + if (private) { + bool free = false; + + private->owner_task = current; + + spin_lock(&inode->lock); + if (file->private_data) + free = true; + else + file->private_data = private; + spin_unlock(&inode->lock); + + if (free) { + kfree(private); + private = NULL; + } + } } if (private) From b0b595e61d97de61c15b379b754b2caa90e83e5c Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Wed, 11 Sep 2024 07:06:45 +0930 Subject: [PATCH 215/302] btrfs: tree-checker: fix the wrong output of data backref objectid [BUG] There are some reports about invalid data backref objectids, the report looks like this: BTRFS critical (device sda): corrupt leaf: block=333654787489792 slot=110 extent bytenr=333413935558656 len=65536 invalid data ref objectid value 2543 The data ref objectid is the inode number inside the subvolume. But in above case, the value is completely sane, not really showing the problem. [CAUSE] The root cause of the problem is the deprecated feature, inode cache. This feature results a special inode number, -12ULL, and it's no longer recognized by tree-checker, triggering the error. The direct problem here is the output of data ref objectid. The value shown is in fact the dref_root (subvolume id), not the dref_objectid (inode number). [FIX] Fix the output to use dref_objectid instead. Reported-by: Neil Parton Reported-by: Archange Link: https://lore.kernel.org/linux-btrfs/CAAYHqBbrrgmh6UmW3ANbysJX9qG9Pbg3ZwnKsV=5mOpv_qix_Q@mail.gmail.com/ Link: https://lore.kernel.org/linux-btrfs/9541deea-9056-406e-be16-a996b549614d@archlinux.org/ Fixes: f333a3c7e832 ("btrfs: tree-checker: validate dref root and objectid") CC: stable@vger.kernel.org # 6.11 Reviewed-by: Filipe Manana Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/tree-checker.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/btrfs/tree-checker.c b/fs/btrfs/tree-checker.c index 634d69964fe4..7b50263723bc 100644 --- a/fs/btrfs/tree-checker.c +++ b/fs/btrfs/tree-checker.c @@ -1517,7 +1517,7 @@ static int check_extent_item(struct extent_buffer *leaf, dref_objectid > BTRFS_LAST_FREE_OBJECTID)) { extent_err(leaf, slot, "invalid data ref objectid value %llu", - dref_root); + dref_objectid); return -EUCLEAN; } if (unlikely(!IS_ALIGNED(dref_offset, From 7f1b63f981b8284c6d8238cb49b5cb156d9a833e Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Sun, 15 Sep 2024 20:52:53 +0100 Subject: [PATCH 216/302] btrfs: fix use-after-free on rbtree that tracks inodes for auto defrag When cleaning up defrag inodes at btrfs_cleanup_defrag_inodes(), called during remount and unmount, we are freeing every node from the rbtree that tracks inodes for auto defrag using rbtree_postorder_for_each_entry_safe(), which doesn't modify the tree itself. So once we unlock the lock that protects the rbtree, we have a tree pointing to a root that was freed (and a root pointing to freed nodes, and their children pointing to other freed nodes, and so on). This makes further access to the tree result in a use-after-free with unpredictable results. Fix this by initializing the rbtree to an empty root after the call to rbtree_postorder_for_each_entry_safe() and before unlocking. Fixes: 276940915f23 ("btrfs: clear defragmented inodes using postorder in btrfs_cleanup_defrag_inodes()") Reported-by: syzbot+ad7966ca1f5dd8b001b3@syzkaller.appspotmail.com Link: https://lore.kernel.org/linux-btrfs/000000000000f9aad406223eabff@google.com/ Reviewed-by: Qu Wenruo Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/defrag.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fs/btrfs/defrag.c b/fs/btrfs/defrag.c index acf1f39e45d0..b95ef44c326b 100644 --- a/fs/btrfs/defrag.c +++ b/fs/btrfs/defrag.c @@ -213,6 +213,8 @@ void btrfs_cleanup_defrag_inodes(struct btrfs_fs_info *fs_info) &fs_info->defrag_inodes, rb_node) kmem_cache_free(btrfs_inode_defrag_cachep, defrag); + fs_info->defrag_inodes = RB_ROOT; + spin_unlock(&fs_info->defrag_inodes_lock); } From 5c7bdac783be8dcba1427460e7971445f839a5e2 Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Thu, 14 Mar 2024 14:54:46 -0500 Subject: [PATCH 217/302] PCI: Fix typos Fix typos. Signed-off-by: Bjorn Helgaas --- drivers/pci/ats.c | 4 ++-- drivers/pci/controller/cadence/Kconfig | 2 +- drivers/pci/hotplug/cpqphp_core.c | 2 +- drivers/pci/hotplug/s390_pci_hpc.c | 2 +- drivers/pci/iomap.c | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/pci/ats.c b/drivers/pci/ats.c index c570892b2090..956b08117347 100644 --- a/drivers/pci/ats.c +++ b/drivers/pci/ats.c @@ -455,8 +455,8 @@ void pci_restore_pasid_state(struct pci_dev *pdev) * pci_pasid_features - Check which PASID features are supported * @pdev: PCI device structure * - * Returns a negative value when no PASI capability is present. - * Otherwise is returns a bitmask with supported features. Current + * Return a negative value when no PASID capability is present. + * Otherwise return a bitmask with supported features. Current * features reported are: * PCI_PASID_CAP_EXEC - Execute permission supported * PCI_PASID_CAP_PRIV - Privileged mode supported diff --git a/drivers/pci/controller/cadence/Kconfig b/drivers/pci/controller/cadence/Kconfig index 1d5a70c9055e..8a0044bb3989 100644 --- a/drivers/pci/controller/cadence/Kconfig +++ b/drivers/pci/controller/cadence/Kconfig @@ -38,7 +38,7 @@ config PCIE_CADENCE_PLAT_EP select PCIE_CADENCE_EP select PCIE_CADENCE_PLAT help - Say Y here if you want to support the Cadence PCIe platform controller in + Say Y here if you want to support the Cadence PCIe platform controller in endpoint mode. This PCIe controller may be embedded into many different vendors SoCs. diff --git a/drivers/pci/hotplug/cpqphp_core.c b/drivers/pci/hotplug/cpqphp_core.c index c94b40e64baf..47a3ed16159a 100644 --- a/drivers/pci/hotplug/cpqphp_core.c +++ b/drivers/pci/hotplug/cpqphp_core.c @@ -328,7 +328,7 @@ get_slot_mapping(struct pci_bus *bus, u8 bus_num, u8 dev_num, u8 *slot) } else { /* Did not get a match on the target PCI device. Check * if the current IRQ table entry is a PCI-to-PCI - * bridge device. If so, and it's secondary bus + * bridge device. If so, and its secondary bus * matches the bus number for the target device, I need * to save the bridge's slot number. If I can not find * an entry for the target device, I will have to diff --git a/drivers/pci/hotplug/s390_pci_hpc.c b/drivers/pci/hotplug/s390_pci_hpc.c index 7333b305f2a5..055518ee354d 100644 --- a/drivers/pci/hotplug/s390_pci_hpc.c +++ b/drivers/pci/hotplug/s390_pci_hpc.c @@ -112,7 +112,7 @@ static int get_power_status(struct hotplug_slot *hotplug_slot, u8 *value) static int get_adapter_status(struct hotplug_slot *hotplug_slot, u8 *value) { - /* if the slot exits it always contains a function */ + /* if the slot exists it always contains a function */ *value = 1; return 0; } diff --git a/drivers/pci/iomap.c b/drivers/pci/iomap.c index a715a4803c95..9fb7cacc15cd 100644 --- a/drivers/pci/iomap.c +++ b/drivers/pci/iomap.c @@ -156,7 +156,7 @@ EXPORT_SYMBOL_GPL(pci_iomap_wc); * the different IOMAP ranges. * * But if the architecture does not use the generic iomap code, and if - * it has _not_ defined it's own private pci_iounmap function, we define + * it has _not_ defined its own private pci_iounmap function, we define * it here. * * NOTE! This default implementation assumes that if the architecture From 73598a0cfb21fb21928e638f7f21be6021ac2a16 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Tue, 6 Aug 2024 21:11:32 -0400 Subject: [PATCH 218/302] nfsd: don't allocate the versions array. Instead of using kmalloc to allocate an array for storing active version info, just declare an array to the max size - it is only 5 or so. Signed-off-by: NeilBrown Signed-off-by: Chuck Lever --- fs/nfsd/cache.h | 2 +- fs/nfsd/netns.h | 6 +-- fs/nfsd/nfsctl.c | 10 +++-- fs/nfsd/nfsd.h | 9 ++++- fs/nfsd/nfssvc.c | 102 +++++++++-------------------------------------- 5 files changed, 35 insertions(+), 94 deletions(-) diff --git a/fs/nfsd/cache.h b/fs/nfsd/cache.h index 66a05fefae98..bb7addef4a31 100644 --- a/fs/nfsd/cache.h +++ b/fs/nfsd/cache.h @@ -10,7 +10,7 @@ #define NFSCACHE_H #include -#include "netns.h" +#include "nfsd.h" /* * Representation of a reply cache entry. diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h index 14ec15656320..238fc4e56e53 100644 --- a/fs/nfsd/netns.h +++ b/fs/nfsd/netns.h @@ -152,8 +152,8 @@ struct nfsd_net { /* * Version information */ - bool *nfsd_versions; - bool *nfsd4_minorversions; + bool nfsd_versions[NFSD_MAXVERS + 1]; + bool nfsd4_minorversions[NFSD_SUPPORTED_MINOR_VERSION + 1]; /* * Duplicate reply cache @@ -219,8 +219,6 @@ struct nfsd_net { #define nfsd_netns_ready(nn) ((nn)->sessionid_hashtbl) extern bool nfsd_support_version(int vers); -extern void nfsd_netns_free_versions(struct nfsd_net *nn); - extern unsigned int nfsd_net_id; void nfsd_copy_write_verifier(__be32 verf[2], struct nfsd_net *nn); diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c index 97c8c76b241b..965519995aeb 100644 --- a/fs/nfsd/nfsctl.c +++ b/fs/nfsd/nfsctl.c @@ -2231,8 +2231,9 @@ err_free_msg: */ static __net_init int nfsd_net_init(struct net *net) { - int retval; struct nfsd_net *nn = net_generic(net, nfsd_net_id); + int retval; + int i; retval = nfsd_export_init(net); if (retval) @@ -2246,8 +2247,10 @@ static __net_init int nfsd_net_init(struct net *net) goto out_repcache_error; memset(&nn->nfsd_svcstats, 0, sizeof(nn->nfsd_svcstats)); nn->nfsd_svcstats.program = &nfsd_program; - nn->nfsd_versions = NULL; - nn->nfsd4_minorversions = NULL; + for (i = 0; i < sizeof(nn->nfsd_versions); i++) + nn->nfsd_versions[i] = nfsd_support_version(i); + for (i = 0; i < sizeof(nn->nfsd4_minorversions); i++) + nn->nfsd4_minorversions[i] = nfsd_support_version(4); nn->nfsd_info.mutex = &nfsd_mutex; nn->nfsd_serv = NULL; nfsd4_init_leases_net(nn); @@ -2278,7 +2281,6 @@ static __net_exit void nfsd_net_exit(struct net *net) percpu_counter_destroy_many(nn->counter, NFSD_STATS_COUNTERS_NUM); nfsd_idmap_shutdown(net); nfsd_export_shutdown(net); - nfsd_netns_free_versions(nn); } static struct pernet_operations nfsd_net_ops = { diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h index 39e109a7d56d..369c3b3ce53e 100644 --- a/fs/nfsd/nfsd.h +++ b/fs/nfsd/nfsd.h @@ -23,9 +23,7 @@ #include -#include "netns.h" #include "export.h" -#include "stats.h" #undef ifdebug #ifdef CONFIG_SUNRPC_DEBUG @@ -37,7 +35,14 @@ /* * nfsd version */ +#define NFSD_MINVERS 2 +#define NFSD_MAXVERS 4 #define NFSD_SUPPORTED_MINOR_VERSION 2 +bool nfsd_support_version(int vers); + +#include "netns.h" +#include "stats.h" + /* * Maximum blocksizes supported by daemon under various circumstances. */ diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c index f25b26bc5670..5bc4dc60ea4c 100644 --- a/fs/nfsd/nfssvc.c +++ b/fs/nfsd/nfssvc.c @@ -106,7 +106,7 @@ static struct svc_program nfsd_acl_program = { #endif /* defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL) */ -static const struct svc_version *nfsd_version[] = { +static const struct svc_version *nfsd_version[NFSD_MAXVERS+1] = { #if defined(CONFIG_NFSD_V2) [2] = &nfsd_version2, #endif @@ -116,15 +116,12 @@ static const struct svc_version *nfsd_version[] = { #endif }; -#define NFSD_MINVERS 2 -#define NFSD_NRVERS ARRAY_SIZE(nfsd_version) - struct svc_program nfsd_program = { #if defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL) .pg_next = &nfsd_acl_program, #endif .pg_prog = NFS_PROGRAM, /* program number */ - .pg_nvers = NFSD_NRVERS, /* nr of entries in nfsd_version */ + .pg_nvers = NFSD_MAXVERS+1, /* nr of entries in nfsd_version */ .pg_vers = nfsd_version, /* version table */ .pg_name = "nfsd", /* program name */ .pg_class = "nfsd", /* authentication class */ @@ -135,78 +132,24 @@ struct svc_program nfsd_program = { bool nfsd_support_version(int vers) { - if (vers >= NFSD_MINVERS && vers < NFSD_NRVERS) + if (vers >= NFSD_MINVERS && vers <= NFSD_MAXVERS) return nfsd_version[vers] != NULL; return false; } -static bool * -nfsd_alloc_versions(void) -{ - bool *vers = kmalloc_array(NFSD_NRVERS, sizeof(bool), GFP_KERNEL); - unsigned i; - - if (vers) { - /* All compiled versions are enabled by default */ - for (i = 0; i < NFSD_NRVERS; i++) - vers[i] = nfsd_support_version(i); - } - return vers; -} - -static bool * -nfsd_alloc_minorversions(void) -{ - bool *vers = kmalloc_array(NFSD_SUPPORTED_MINOR_VERSION + 1, - sizeof(bool), GFP_KERNEL); - unsigned i; - - if (vers) { - /* All minor versions are enabled by default */ - for (i = 0; i <= NFSD_SUPPORTED_MINOR_VERSION; i++) - vers[i] = nfsd_support_version(4); - } - return vers; -} - -void -nfsd_netns_free_versions(struct nfsd_net *nn) -{ - kfree(nn->nfsd_versions); - kfree(nn->nfsd4_minorversions); - nn->nfsd_versions = NULL; - nn->nfsd4_minorversions = NULL; -} - -static void -nfsd_netns_init_versions(struct nfsd_net *nn) -{ - if (!nn->nfsd_versions) { - nn->nfsd_versions = nfsd_alloc_versions(); - nn->nfsd4_minorversions = nfsd_alloc_minorversions(); - if (!nn->nfsd_versions || !nn->nfsd4_minorversions) - nfsd_netns_free_versions(nn); - } -} - int nfsd_vers(struct nfsd_net *nn, int vers, enum vers_op change) { - if (vers < NFSD_MINVERS || vers >= NFSD_NRVERS) + if (vers < NFSD_MINVERS || vers > NFSD_MAXVERS) return 0; switch(change) { case NFSD_SET: - if (nn->nfsd_versions) - nn->nfsd_versions[vers] = nfsd_support_version(vers); + nn->nfsd_versions[vers] = nfsd_support_version(vers); break; case NFSD_CLEAR: - nfsd_netns_init_versions(nn); - if (nn->nfsd_versions) - nn->nfsd_versions[vers] = false; + nn->nfsd_versions[vers] = false; break; case NFSD_TEST: - if (nn->nfsd_versions) - return nn->nfsd_versions[vers]; - fallthrough; + return nn->nfsd_versions[vers]; case NFSD_AVAIL: return nfsd_support_version(vers); } @@ -233,23 +176,16 @@ int nfsd_minorversion(struct nfsd_net *nn, u32 minorversion, enum vers_op change switch(change) { case NFSD_SET: - if (nn->nfsd4_minorversions) { - nfsd_vers(nn, 4, NFSD_SET); - nn->nfsd4_minorversions[minorversion] = - nfsd_vers(nn, 4, NFSD_TEST); - } + nfsd_vers(nn, 4, NFSD_SET); + nn->nfsd4_minorversions[minorversion] = + nfsd_vers(nn, 4, NFSD_TEST); break; case NFSD_CLEAR: - nfsd_netns_init_versions(nn); - if (nn->nfsd4_minorversions) { - nn->nfsd4_minorversions[minorversion] = false; - nfsd_adjust_nfsd_versions4(nn); - } + nn->nfsd4_minorversions[minorversion] = false; + nfsd_adjust_nfsd_versions4(nn); break; case NFSD_TEST: - if (nn->nfsd4_minorversions) - return nn->nfsd4_minorversions[minorversion]; - return nfsd_vers(nn, 4, NFSD_TEST); + return nn->nfsd4_minorversions[minorversion]; case NFSD_AVAIL: return minorversion <= NFSD_SUPPORTED_MINOR_VERSION && nfsd_vers(nn, 4, NFSD_AVAIL); @@ -568,11 +504,11 @@ void nfsd_reset_versions(struct nfsd_net *nn) { int i; - for (i = 0; i < NFSD_NRVERS; i++) + for (i = 0; i <= NFSD_MAXVERS; i++) if (nfsd_vers(nn, i, NFSD_TEST)) return; - for (i = 0; i < NFSD_NRVERS; i++) + for (i = 0; i <= NFSD_MAXVERS; i++) if (i != 4) nfsd_vers(nn, i, NFSD_SET); else { @@ -905,17 +841,17 @@ nfsd_init_request(struct svc_rqst *rqstp, if (likely(nfsd_vers(nn, rqstp->rq_vers, NFSD_TEST))) return svc_generic_init_request(rqstp, progp, ret); - ret->mismatch.lovers = NFSD_NRVERS; - for (i = NFSD_MINVERS; i < NFSD_NRVERS; i++) { + ret->mismatch.lovers = NFSD_MAXVERS + 1; + for (i = NFSD_MINVERS; i <= NFSD_MAXVERS; i++) { if (nfsd_vers(nn, i, NFSD_TEST)) { ret->mismatch.lovers = i; break; } } - if (ret->mismatch.lovers == NFSD_NRVERS) + if (ret->mismatch.lovers > NFSD_MAXVERS) return rpc_prog_unavail; ret->mismatch.hivers = NFSD_MINVERS; - for (i = NFSD_NRVERS - 1; i >= NFSD_MINVERS; i--) { + for (i = NFSD_MAXVERS; i >= NFSD_MINVERS; i--) { if (nfsd_vers(nn, i, NFSD_TEST)) { ret->mismatch.hivers = i; break; From 16ef80eedcd34799db69990e13f69b812d2690f1 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Fri, 26 Jul 2024 13:59:55 +1000 Subject: [PATCH 219/302] sunrpc: document locking rules for svc_exit_thread() The locking required for svc_exit_thread() is not obvious, so document it in a kdoc comment. Signed-off-by: NeilBrown Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- net/sunrpc/svc.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index 561d20a5316e..2e148137376d 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -945,6 +945,20 @@ void svc_rqst_release_pages(struct svc_rqst *rqstp) } } +/** + * svc_exit_thread - finalise the termination of a sunrpc server thread + * @rqstp: the svc_rqst which represents the thread. + * + * When a thread started with svc_new_thread() exits it must call + * svc_exit_thread() as its last act. This must be done with the + * service mutex held. Normally this is held by a DIFFERENT thread, the + * one that is calling svc_set_num_threads() and which will wait for + * SP_VICTIM_REMAINS to be cleared before dropping the mutex. If the + * thread exits for any reason other than svc_thread_should_stop() + * returning %true (which indicated that svc_set_num_threads() is + * waiting for it to exit), then it must take the service mutex itself, + * which can only safely be done using mutex_try_lock(). + */ void svc_exit_thread(struct svc_rqst *rqstp) { From 60749cbe3d8ae572a6c7dda675de3e8b25797a18 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Mon, 15 Jul 2024 17:14:18 +1000 Subject: [PATCH 220/302] sunrpc: change sp_nrthreads from atomic_t to unsigned int. sp_nrthreads is only ever accessed under the service mutex nlmsvc_mutex nfs_callback_mutex nfsd_mutex so these is no need for it to be an atomic_t. The fact that all code using it is single-threaded means that we can simplify svc_pool_victim and remove the temporary elevation of sp_nrthreads. Signed-off-by: NeilBrown Signed-off-by: Chuck Lever --- fs/nfsd/nfsctl.c | 2 +- fs/nfsd/nfssvc.c | 2 +- include/linux/sunrpc/svc.h | 4 ++-- net/sunrpc/svc.c | 31 +++++++++++-------------------- 4 files changed, 15 insertions(+), 24 deletions(-) diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c index 965519995aeb..1c9e5b4bcb0a 100644 --- a/fs/nfsd/nfsctl.c +++ b/fs/nfsd/nfsctl.c @@ -1769,7 +1769,7 @@ int nfsd_nl_threads_get_doit(struct sk_buff *skb, struct genl_info *info) struct svc_pool *sp = &nn->nfsd_serv->sv_pools[i]; err = nla_put_u32(skb, NFSD_A_SERVER_THREADS, - atomic_read(&sp->sp_nrthreads)); + sp->sp_nrthreads); if (err) goto err_unlock; } diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c index 5bc4dc60ea4c..b1dc3404173b 100644 --- a/fs/nfsd/nfssvc.c +++ b/fs/nfsd/nfssvc.c @@ -641,7 +641,7 @@ int nfsd_get_nrthreads(int n, int *nthreads, struct net *net) if (serv) for (i = 0; i < serv->sv_nrpools && i < n; i++) - nthreads[i] = atomic_read(&serv->sv_pools[i].sp_nrthreads); + nthreads[i] = serv->sv_pools[i].sp_nrthreads; return 0; } diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index e4fa25fafa97..99e9345d829e 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -33,9 +33,9 @@ * node traffic on multi-node NUMA NFS servers. */ struct svc_pool { - unsigned int sp_id; /* pool id; also node id on NUMA */ + unsigned int sp_id; /* pool id; also node id on NUMA */ struct lwq sp_xprts; /* pending transports */ - atomic_t sp_nrthreads; /* # of threads in pool */ + unsigned int sp_nrthreads; /* # of threads in pool */ struct list_head sp_all_threads; /* all server threads */ struct llist_head sp_idle_threads; /* idle server threads */ diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index 2e148137376d..9442ebf38bbd 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -725,7 +725,7 @@ svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node) serv->sv_nrthreads += 1; spin_unlock_bh(&serv->sv_lock); - atomic_inc(&pool->sp_nrthreads); + pool->sp_nrthreads += 1; /* Protected by whatever lock the service uses when calling * svc_set_num_threads() @@ -780,31 +780,22 @@ svc_pool_victim(struct svc_serv *serv, struct svc_pool *target_pool, struct svc_pool *pool; unsigned int i; -retry: pool = target_pool; - if (pool != NULL) { - if (atomic_inc_not_zero(&pool->sp_nrthreads)) - goto found_pool; - return NULL; - } else { + if (!pool) { for (i = 0; i < serv->sv_nrpools; i++) { pool = &serv->sv_pools[--(*state) % serv->sv_nrpools]; - if (atomic_inc_not_zero(&pool->sp_nrthreads)) - goto found_pool; + if (pool->sp_nrthreads) + break; } - return NULL; } -found_pool: - set_bit(SP_VICTIM_REMAINS, &pool->sp_flags); - set_bit(SP_NEED_VICTIM, &pool->sp_flags); - if (!atomic_dec_and_test(&pool->sp_nrthreads)) + if (pool && pool->sp_nrthreads) { + set_bit(SP_VICTIM_REMAINS, &pool->sp_flags); + set_bit(SP_NEED_VICTIM, &pool->sp_flags); return pool; - /* Nothing left in this pool any more */ - clear_bit(SP_NEED_VICTIM, &pool->sp_flags); - clear_bit(SP_VICTIM_REMAINS, &pool->sp_flags); - goto retry; + } + return NULL; } static int @@ -883,7 +874,7 @@ svc_set_num_threads(struct svc_serv *serv, struct svc_pool *pool, int nrservs) if (!pool) nrservs -= serv->sv_nrthreads; else - nrservs -= atomic_read(&pool->sp_nrthreads); + nrservs -= pool->sp_nrthreads; if (nrservs > 0) return svc_start_kthreads(serv, pool, nrservs); @@ -967,7 +958,7 @@ svc_exit_thread(struct svc_rqst *rqstp) list_del_rcu(&rqstp->rq_all); - atomic_dec(&pool->sp_nrthreads); + pool->sp_nrthreads -= 1; spin_lock_bh(&serv->sv_lock); serv->sv_nrthreads -= 1; From 9dcbc4e07087f750010c32b1c56fe1af8792a0ca Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Mon, 15 Jul 2024 17:14:19 +1000 Subject: [PATCH 221/302] sunrpc: don't take ->sv_lock when updating ->sv_nrthreads. As documented in svc_xprt.c, sv_nrthreads is protected by the service mutex, and it does not need ->sv_lock. (->sv_lock is needed only for sv_permsocks, sv_tempsocks, and sv_tmpcnt). So remove the unnecessary locking. Signed-off-by: NeilBrown Signed-off-by: Chuck Lever --- net/sunrpc/svc.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index 9442ebf38bbd..f30eeae56c44 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -721,10 +721,7 @@ svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node) if (!rqstp) return ERR_PTR(-ENOMEM); - spin_lock_bh(&serv->sv_lock); serv->sv_nrthreads += 1; - spin_unlock_bh(&serv->sv_lock); - pool->sp_nrthreads += 1; /* Protected by whatever lock the service uses when calling @@ -959,10 +956,7 @@ svc_exit_thread(struct svc_rqst *rqstp) list_del_rcu(&rqstp->rq_all); pool->sp_nrthreads -= 1; - - spin_lock_bh(&serv->sv_lock); serv->sv_nrthreads -= 1; - spin_unlock_bh(&serv->sv_lock); svc_sock_update_bufs(serv); svc_rqst_free(rqstp); From 59f3b138160d37435b353e95c62d9ebf7f80b117 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Tue, 30 Jul 2024 07:19:41 +1000 Subject: [PATCH 222/302] sunrpc: merge svc_rqst_alloc() into svc_prepare_thread() The only caller of svc_rqst_alloc() is svc_prepare_thread(). So merge the one into the other and simplify. Signed-off-by: NeilBrown Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- net/sunrpc/svc.c | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index f30eeae56c44..17f0f59c068f 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -678,7 +678,7 @@ svc_rqst_free(struct svc_rqst *rqstp) } static struct svc_rqst * -svc_rqst_alloc(struct svc_serv *serv, struct svc_pool *pool, int node) +svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node) { struct svc_rqst *rqstp; @@ -706,21 +706,6 @@ svc_rqst_alloc(struct svc_serv *serv, struct svc_pool *pool, int node) if (!svc_init_buffer(rqstp, serv->sv_max_mesg, node)) goto out_enomem; - return rqstp; -out_enomem: - svc_rqst_free(rqstp); - return NULL; -} - -static struct svc_rqst * -svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node) -{ - struct svc_rqst *rqstp; - - rqstp = svc_rqst_alloc(serv, pool, node); - if (!rqstp) - return ERR_PTR(-ENOMEM); - serv->sv_nrthreads += 1; pool->sp_nrthreads += 1; @@ -730,6 +715,10 @@ svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node) list_add_rcu(&rqstp->rq_all, &pool->sp_all_threads); return rqstp; + +out_enomem: + svc_rqst_free(rqstp); + return NULL; } /** @@ -810,8 +799,8 @@ svc_start_kthreads(struct svc_serv *serv, struct svc_pool *pool, int nrservs) node = svc_pool_map_get_node(chosen_pool->sp_id); rqstp = svc_prepare_thread(serv, chosen_pool, node); - if (IS_ERR(rqstp)) - return PTR_ERR(rqstp); + if (!rqstp) + return -ENOMEM; task = kthread_create_on_node(serv->sv_threadfn, rqstp, node, "%s", serv->sv_name); if (IS_ERR(task)) { From 3391fc92db8e761f1a2df5612fcb999dac6bc00a Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Mon, 16 Sep 2024 09:45:40 +1000 Subject: [PATCH 223/302] sunrpc: allow svc threads to fail initialisation cleanly If an svc thread needs to perform some initialisation that might fail, it has no good way to handle the failure. Before the thread can exit it must call svc_exit_thread(), but that requires the service mutex to be held. The thread cannot simply take the mutex as that could deadlock if there is a concurrent attempt to shut down all threads (which is unlikely, but not impossible). nfsd currently call svc_exit_thread() unprotected in the unlikely event that unshare_fs_struct() fails. We can clean this up by introducing svc_thread_init_status() by which an svc thread can report whether initialisation has succeeded. If it has, it continues normally into the action loop. If it has not, svc_thread_init_status() immediately aborts the thread. svc_start_kthread() waits for either of these to happen, and calls svc_exit_thread() (under the mutex) if the thread aborted. Signed-off-by: NeilBrown Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/lockd/svc.c | 2 ++ fs/nfs/callback.c | 2 ++ fs/nfsd/nfssvc.c | 9 +++------ include/linux/sunrpc/svc.h | 31 +++++++++++++++++++++++++++++++ net/sunrpc/svc.c | 10 ++++++++++ 5 files changed, 48 insertions(+), 6 deletions(-) diff --git a/fs/lockd/svc.c b/fs/lockd/svc.c index 71713309967d..4ec22c2f2ea3 100644 --- a/fs/lockd/svc.c +++ b/fs/lockd/svc.c @@ -124,6 +124,8 @@ lockd(void *vrqstp) struct net *net = &init_net; struct lockd_net *ln = net_generic(net, lockd_net_id); + svc_thread_init_status(rqstp, 0); + /* try_to_freeze() is called from svc_recv() */ set_freezable(); diff --git a/fs/nfs/callback.c b/fs/nfs/callback.c index 8adfcd4c8c1a..6cf92498a5ac 100644 --- a/fs/nfs/callback.c +++ b/fs/nfs/callback.c @@ -76,6 +76,8 @@ nfs4_callback_svc(void *vrqstp) { struct svc_rqst *rqstp = vrqstp; + svc_thread_init_status(rqstp, 0); + set_freezable(); while (!svc_thread_should_stop(rqstp)) diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c index b1dc3404173b..3fb6c8c9a2f0 100644 --- a/fs/nfsd/nfssvc.c +++ b/fs/nfsd/nfssvc.c @@ -873,11 +873,9 @@ nfsd(void *vrqstp) /* At this point, the thread shares current->fs * with the init process. We need to create files with the - * umask as defined by the client instead of init's umask. */ - if (unshare_fs_struct() < 0) { - printk("Unable to start nfsd thread: out of memory\n"); - goto out; - } + * umask as defined by the client instead of init's umask. + */ + svc_thread_init_status(rqstp, unshare_fs_struct()); current->fs->umask = 0; @@ -899,7 +897,6 @@ nfsd(void *vrqstp) atomic_dec(&nfsd_th_cnt); -out: /* Release the thread */ svc_exit_thread(rqstp); return 0; diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index 99e9345d829e..c419a61f60e5 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -21,6 +21,7 @@ #include #include #include +#include /* * @@ -232,6 +233,11 @@ struct svc_rqst { struct net *rq_bc_net; /* pointer to backchannel's * net namespace */ + + int rq_err; /* Thread sets this to inidicate + * initialisation success. + */ + unsigned long bc_to_initval; unsigned int bc_to_retries; void ** rq_lease_breaker; /* The v4 client breaking a lease */ @@ -305,6 +311,31 @@ static inline bool svc_thread_should_stop(struct svc_rqst *rqstp) return test_bit(RQ_VICTIM, &rqstp->rq_flags); } +/** + * svc_thread_init_status - report whether thread has initialised successfully + * @rqstp: the thread in question + * @err: errno code + * + * After performing any initialisation that could fail, and before starting + * normal work, each sunrpc svc_thread must call svc_thread_init_status() + * with an appropriate error, or zero. + * + * If zero is passed, the thread is ready and must continue until + * svc_thread_should_stop() returns true. If a non-zero error is passed + * the call will not return - the thread will exit. + */ +static inline void svc_thread_init_status(struct svc_rqst *rqstp, int err) +{ + rqstp->rq_err = err; + /* memory barrier ensures assignment to error above is visible before + * waitqueue_active() test below completes. + */ + smp_mb(); + wake_up_var(&rqstp->rq_err); + if (err) + kthread_exit(1); +} + struct svc_deferred_req { u32 prot; /* protocol (UDP or TCP) */ struct svc_xprt *xprt; diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index 17f0f59c068f..9aff845196ce 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -706,6 +706,8 @@ svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node) if (!svc_init_buffer(rqstp, serv->sv_max_mesg, node)) goto out_enomem; + rqstp->rq_err = -EAGAIN; /* No error yet */ + serv->sv_nrthreads += 1; pool->sp_nrthreads += 1; @@ -792,6 +794,7 @@ svc_start_kthreads(struct svc_serv *serv, struct svc_pool *pool, int nrservs) struct svc_pool *chosen_pool; unsigned int state = serv->sv_nrthreads-1; int node; + int err; do { nrservs--; @@ -814,6 +817,13 @@ svc_start_kthreads(struct svc_serv *serv, struct svc_pool *pool, int nrservs) svc_sock_update_bufs(serv); wake_up_process(task); + + wait_var_event(&rqstp->rq_err, rqstp->rq_err != -EAGAIN); + err = rqstp->rq_err; + if (err) { + svc_exit_thread(rqstp); + return err; + } } while (nrservs > 0); return 0; From 11673b2a917d2351d8e29d4788a129c973852215 Mon Sep 17 00:00:00 2001 From: Sagi Grimberg Date: Wed, 24 Jul 2024 10:01:37 -0700 Subject: [PATCH 224/302] nfsd: don't assume copy notify when preprocessing the stateid Move the stateid handling to nfsd4_copy_notify. If nfs4_preprocess_stateid_op did not produce an output stateid, error out. Copy notify specifically does not permit the use of special stateids, so enforce that outside generic stateid pre-processing. Signed-off-by: Sagi Grimberg Reviewed-by: Jeff Layton Reviewed-by: Olga Kornievskaia Signed-off-by: Chuck Lever --- fs/nfsd/nfs4proc.c | 4 +++- fs/nfsd/nfs4state.c | 6 +----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index 2e39cf2e502a..911134ed5a0e 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -1942,7 +1942,7 @@ nfsd4_copy_notify(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, struct nfsd4_copy_notify *cn = &u->copy_notify; __be32 status; struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); - struct nfs4_stid *stid; + struct nfs4_stid *stid = NULL; struct nfs4_cpntf_state *cps; struct nfs4_client *clp = cstate->clp; @@ -1951,6 +1951,8 @@ nfsd4_copy_notify(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, &stid); if (status) return status; + if (!stid) + return nfserr_bad_stateid; cn->cpn_lease_time.tv_sec = nn->nfsd4_lease; cn->cpn_lease_time.tv_nsec = 0; diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 5c48915ff61c..8fddc2d1c80e 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -7022,11 +7022,7 @@ nfs4_preprocess_stateid_op(struct svc_rqst *rqstp, *nfp = NULL; if (ZERO_STATEID(stateid) || ONE_STATEID(stateid)) { - if (cstid) - status = nfserr_bad_stateid; - else - status = check_special_stateids(net, fhp, stateid, - flags); + status = check_special_stateids(net, fhp, stateid, flags); goto done; } From c55aeef7766cdc52847d03dd1326859c55811a52 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Fri, 26 Jul 2024 12:21:30 +1000 Subject: [PATCH 225/302] nfsd: Don't pass all of rqst into rqst_exp_find() Rather than passing the whole rqst, pass the pieces that are actually needed. This makes the inputs to rqst_exp_find() more obvious. Signed-off-by: NeilBrown Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/export.c | 35 ++++++++++++++++++++++++++--------- fs/nfsd/export.h | 4 +++- fs/nfsd/nfs4proc.c | 4 +++- fs/nfsd/nfsfh.c | 4 +++- 4 files changed, 35 insertions(+), 12 deletions(-) diff --git a/fs/nfsd/export.c b/fs/nfsd/export.c index 50b3135d07ac..9aa5f95f18a8 100644 --- a/fs/nfsd/export.c +++ b/fs/nfsd/export.c @@ -1164,19 +1164,35 @@ gss: return gssexp; } +/** + * rqst_exp_find - Find an svc_export in the context of a rqst or similar + * @reqp: The handle to be used to suspend the request if a cache-upcall is needed + * If NULL, missing in-cache information will result in failure. + * @net: The network namespace in which the request exists + * @cl: default auth_domain to use for looking up the export + * @gsscl: an alternate auth_domain defined using deprecated gss/krb5 format. + * @fsid_type: The type of fsid to look for + * @fsidv: The actual fsid to look up in the context of either client. + * + * Perform a lookup for @cl/@fsidv in the given @net for an export. If + * none found and @gsscl specified, repeat the lookup. + * + * Returns an export, or an error pointer. + */ struct svc_export * -rqst_exp_find(struct svc_rqst *rqstp, int fsid_type, u32 *fsidv) +rqst_exp_find(struct cache_req *reqp, struct net *net, + struct auth_domain *cl, struct auth_domain *gsscl, + int fsid_type, u32 *fsidv) { + struct nfsd_net *nn = net_generic(net, nfsd_net_id); struct svc_export *gssexp, *exp = ERR_PTR(-ENOENT); - struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); struct cache_detail *cd = nn->svc_export_cache; - if (rqstp->rq_client == NULL) + if (!cl) goto gss; /* First try the auth_unix client: */ - exp = exp_find(cd, rqstp->rq_client, fsid_type, - fsidv, &rqstp->rq_chandle); + exp = exp_find(cd, cl, fsid_type, fsidv, reqp); if (PTR_ERR(exp) == -ENOENT) goto gss; if (IS_ERR(exp)) @@ -1186,10 +1202,9 @@ rqst_exp_find(struct svc_rqst *rqstp, int fsid_type, u32 *fsidv) return exp; gss: /* Otherwise, try falling back on gss client */ - if (rqstp->rq_gssclient == NULL) + if (!gsscl) return exp; - gssexp = exp_find(cd, rqstp->rq_gssclient, fsid_type, fsidv, - &rqstp->rq_chandle); + gssexp = exp_find(cd, gsscl, fsid_type, fsidv, reqp); if (PTR_ERR(gssexp) == -ENOENT) return exp; if (!IS_ERR(exp)) @@ -1220,7 +1235,9 @@ struct svc_export *rqst_find_fsidzero_export(struct svc_rqst *rqstp) mk_fsid(FSID_NUM, fsidv, 0, 0, 0, NULL); - return rqst_exp_find(rqstp, FSID_NUM, fsidv); + return rqst_exp_find(&rqstp->rq_chandle, SVC_NET(rqstp), + rqstp->rq_client, rqstp->rq_gssclient, + FSID_NUM, fsidv); } /* diff --git a/fs/nfsd/export.h b/fs/nfsd/export.h index ca9dc230ae3d..cb17f05e3329 100644 --- a/fs/nfsd/export.h +++ b/fs/nfsd/export.h @@ -127,6 +127,8 @@ static inline struct svc_export *exp_get(struct svc_export *exp) cache_get(&exp->h); return exp; } -struct svc_export * rqst_exp_find(struct svc_rqst *, int, u32 *); +struct svc_export *rqst_exp_find(struct cache_req *reqp, struct net *net, + struct auth_domain *cl, struct auth_domain *gsscl, + int fsid_type, u32 *fsidv); #endif /* NFSD_EXPORT_H */ diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index 911134ed5a0e..4f5b8c4be417 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -2233,7 +2233,9 @@ nfsd4_getdeviceinfo(struct svc_rqst *rqstp, return nfserr_noent; } - exp = rqst_exp_find(rqstp, map->fsid_type, map->fsid); + exp = rqst_exp_find(&rqstp->rq_chandle, SVC_NET(rqstp), + rqstp->rq_client, rqstp->rq_gssclient, + map->fsid_type, map->fsid); if (IS_ERR(exp)) { dprintk("%s: could not find device id\n", __func__); return nfserr_noent; diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c index dd4e11a703aa..17d40600e02c 100644 --- a/fs/nfsd/nfsfh.c +++ b/fs/nfsd/nfsfh.c @@ -195,7 +195,9 @@ static __be32 nfsd_set_fh_dentry(struct svc_rqst *rqstp, struct svc_fh *fhp) data_left -= len; if (data_left < 0) return error; - exp = rqst_exp_find(rqstp, fh->fh_fsid_type, fh->fh_fsid); + exp = rqst_exp_find(&rqstp->rq_chandle, SVC_NET(rqstp), + rqstp->rq_client, rqstp->rq_gssclient, + fh->fh_fsid_type, fh->fh_fsid); fid = (struct fid *)(fh->fh_fsid + len); error = nfserr_stale; From 9fd45c16f3e3b95b458e049c77f4d7dfef673a52 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Fri, 26 Jul 2024 12:21:31 +1000 Subject: [PATCH 226/302] nfsd: Pass 'cred' instead of 'rqstp' to some functions. nfsd_permission(), exp_rdonly(), nfsd_setuser(), and nfsexp_flags() only ever need the cred out of rqstp, so pass it explicitly instead of the whole rqstp. This makes the interfaces cleaner. Signed-off-by: NeilBrown Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/auth.c | 14 +++++++------- fs/nfsd/auth.h | 2 +- fs/nfsd/export.h | 3 ++- fs/nfsd/nfs4state.c | 3 ++- fs/nfsd/nfsfh.c | 6 +++--- fs/nfsd/nfsproc.c | 9 +++++---- fs/nfsd/vfs.c | 21 ++++++++++++--------- fs/nfsd/vfs.h | 4 ++-- 8 files changed, 34 insertions(+), 28 deletions(-) diff --git a/fs/nfsd/auth.c b/fs/nfsd/auth.c index e6beaaf4f170..93e33d1ee891 100644 --- a/fs/nfsd/auth.c +++ b/fs/nfsd/auth.c @@ -5,26 +5,26 @@ #include "nfsd.h" #include "auth.h" -int nfsexp_flags(struct svc_rqst *rqstp, struct svc_export *exp) +int nfsexp_flags(struct svc_cred *cred, struct svc_export *exp) { struct exp_flavor_info *f; struct exp_flavor_info *end = exp->ex_flavors + exp->ex_nflavors; for (f = exp->ex_flavors; f < end; f++) { - if (f->pseudoflavor == rqstp->rq_cred.cr_flavor) + if (f->pseudoflavor == cred->cr_flavor) return f->flags; } return exp->ex_flags; } -int nfsd_setuser(struct svc_rqst *rqstp, struct svc_export *exp) +int nfsd_setuser(struct svc_cred *cred, struct svc_export *exp) { struct group_info *rqgi; struct group_info *gi; struct cred *new; int i; - int flags = nfsexp_flags(rqstp, exp); + int flags = nfsexp_flags(cred, exp); /* discard any old override before preparing the new set */ revert_creds(get_cred(current_real_cred())); @@ -32,10 +32,10 @@ int nfsd_setuser(struct svc_rqst *rqstp, struct svc_export *exp) if (!new) return -ENOMEM; - new->fsuid = rqstp->rq_cred.cr_uid; - new->fsgid = rqstp->rq_cred.cr_gid; + new->fsuid = cred->cr_uid; + new->fsgid = cred->cr_gid; - rqgi = rqstp->rq_cred.cr_group_info; + rqgi = cred->cr_group_info; if (flags & NFSEXP_ALLSQUASH) { new->fsuid = exp->ex_anon_uid; diff --git a/fs/nfsd/auth.h b/fs/nfsd/auth.h index dbd66424f600..8c5031bbbcee 100644 --- a/fs/nfsd/auth.h +++ b/fs/nfsd/auth.h @@ -12,6 +12,6 @@ * Set the current process's fsuid/fsgid etc to those of the NFS * client user */ -int nfsd_setuser(struct svc_rqst *, struct svc_export *); +int nfsd_setuser(struct svc_cred *cred, struct svc_export *exp); #endif /* LINUX_NFSD_AUTH_H */ diff --git a/fs/nfsd/export.h b/fs/nfsd/export.h index cb17f05e3329..3794ae253a70 100644 --- a/fs/nfsd/export.h +++ b/fs/nfsd/export.h @@ -99,7 +99,8 @@ struct svc_expkey { #define EX_NOHIDE(exp) ((exp)->ex_flags & NFSEXP_NOHIDE) #define EX_WGATHER(exp) ((exp)->ex_flags & NFSEXP_GATHERED_WRITES) -int nfsexp_flags(struct svc_rqst *rqstp, struct svc_export *exp); +struct svc_cred; +int nfsexp_flags(struct svc_cred *cred, struct svc_export *exp); __be32 check_nfsd_access(struct svc_export *exp, struct svc_rqst *rqstp); /* diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 8fddc2d1c80e..e18b7ef14f59 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -6891,7 +6891,8 @@ nfs4_check_file(struct svc_rqst *rqstp, struct svc_fh *fhp, struct nfs4_stid *s, nf = nfs4_find_file(s, flags); if (nf) { - status = nfsd_permission(rqstp, fhp->fh_export, fhp->fh_dentry, + status = nfsd_permission(&rqstp->rq_cred, + fhp->fh_export, fhp->fh_dentry, acc | NFSD_MAY_OWNER_OVERRIDE); if (status) { nfsd_file_put(nf); diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c index 17d40600e02c..fffeba082540 100644 --- a/fs/nfsd/nfsfh.c +++ b/fs/nfsd/nfsfh.c @@ -102,7 +102,7 @@ static bool nfsd_originating_port_ok(struct svc_rqst *rqstp, int flags) static __be32 nfsd_setuser_and_check_port(struct svc_rqst *rqstp, struct svc_export *exp) { - int flags = nfsexp_flags(rqstp, exp); + int flags = nfsexp_flags(&rqstp->rq_cred, exp); /* Check if the request originated from a secure port. */ if (!nfsd_originating_port_ok(rqstp, flags)) { @@ -113,7 +113,7 @@ static __be32 nfsd_setuser_and_check_port(struct svc_rqst *rqstp, } /* Set user creds for this exportpoint */ - return nfserrno(nfsd_setuser(rqstp, exp)); + return nfserrno(nfsd_setuser(&rqstp->rq_cred, exp)); } static inline __be32 check_pseudo_root(struct svc_rqst *rqstp, @@ -394,7 +394,7 @@ fh_verify(struct svc_rqst *rqstp, struct svc_fh *fhp, umode_t type, int access) skip_pseudoflavor_check: /* Finally, check access permissions. */ - error = nfsd_permission(rqstp, exp, dentry, access); + error = nfsd_permission(&rqstp->rq_cred, exp, dentry, access); out: trace_nfsd_fh_verify_err(rqstp, fhp, type, access, error); if (error == nfserr_stale) diff --git a/fs/nfsd/nfsproc.c b/fs/nfsd/nfsproc.c index 36370b957b63..97aab34593ef 100644 --- a/fs/nfsd/nfsproc.c +++ b/fs/nfsd/nfsproc.c @@ -331,10 +331,11 @@ nfsd_proc_create(struct svc_rqst *rqstp) * echo thing > device-special-file-or-pipe * by doing a CREATE with type==0 */ - resp->status = nfsd_permission(rqstp, - newfhp->fh_export, - newfhp->fh_dentry, - NFSD_MAY_WRITE|NFSD_MAY_LOCAL_ACCESS); + resp->status = nfsd_permission( + &rqstp->rq_cred, + newfhp->fh_export, + newfhp->fh_dentry, + NFSD_MAY_WRITE|NFSD_MAY_LOCAL_ACCESS); if (resp->status && resp->status != nfserr_rofs) goto out_unlock; } diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c index 29b1f3613800..0862f6ae86a9 100644 --- a/fs/nfsd/vfs.c +++ b/fs/nfsd/vfs.c @@ -421,8 +421,9 @@ nfsd_get_write_access(struct svc_rqst *rqstp, struct svc_fh *fhp, if (iap->ia_size < inode->i_size) { __be32 err; - err = nfsd_permission(rqstp, fhp->fh_export, fhp->fh_dentry, - NFSD_MAY_TRUNC | NFSD_MAY_OWNER_OVERRIDE); + err = nfsd_permission(&rqstp->rq_cred, + fhp->fh_export, fhp->fh_dentry, + NFSD_MAY_TRUNC | NFSD_MAY_OWNER_OVERRIDE); if (err) return err; } @@ -814,7 +815,8 @@ nfsd_access(struct svc_rqst *rqstp, struct svc_fh *fhp, u32 *access, u32 *suppor sresult |= map->access; - err2 = nfsd_permission(rqstp, export, dentry, map->how); + err2 = nfsd_permission(&rqstp->rq_cred, export, + dentry, map->how); switch (err2) { case nfs_ok: result |= map->access; @@ -1475,7 +1477,8 @@ nfsd_create_locked(struct svc_rqst *rqstp, struct svc_fh *fhp, dirp = d_inode(dentry); dchild = dget(resfhp->fh_dentry); - err = nfsd_permission(rqstp, fhp->fh_export, dentry, NFSD_MAY_CREATE); + err = nfsd_permission(&rqstp->rq_cred, fhp->fh_export, dentry, + NFSD_MAY_CREATE); if (err) goto out; @@ -2255,9 +2258,9 @@ nfsd_statfs(struct svc_rqst *rqstp, struct svc_fh *fhp, struct kstatfs *stat, in return err; } -static int exp_rdonly(struct svc_rqst *rqstp, struct svc_export *exp) +static int exp_rdonly(struct svc_cred *cred, struct svc_export *exp) { - return nfsexp_flags(rqstp, exp) & NFSEXP_READONLY; + return nfsexp_flags(cred, exp) & NFSEXP_READONLY; } #ifdef CONFIG_NFSD_V4 @@ -2501,8 +2504,8 @@ out_unlock: * Check for a user's access permissions to this inode. */ __be32 -nfsd_permission(struct svc_rqst *rqstp, struct svc_export *exp, - struct dentry *dentry, int acc) +nfsd_permission(struct svc_cred *cred, struct svc_export *exp, + struct dentry *dentry, int acc) { struct inode *inode = d_inode(dentry); int err; @@ -2533,7 +2536,7 @@ nfsd_permission(struct svc_rqst *rqstp, struct svc_export *exp, */ if (!(acc & NFSD_MAY_LOCAL_ACCESS)) if (acc & (NFSD_MAY_WRITE | NFSD_MAY_SATTR | NFSD_MAY_TRUNC)) { - if (exp_rdonly(rqstp, exp) || + if (exp_rdonly(cred, exp) || __mnt_is_readonly(exp->ex_path.mnt)) return nfserr_rofs; if (/* (acc & NFSD_MAY_WRITE) && */ IS_IMMUTABLE(inode)) diff --git a/fs/nfsd/vfs.h b/fs/nfsd/vfs.h index 57cd70062048..01947561d375 100644 --- a/fs/nfsd/vfs.h +++ b/fs/nfsd/vfs.h @@ -153,8 +153,8 @@ __be32 nfsd_readdir(struct svc_rqst *, struct svc_fh *, __be32 nfsd_statfs(struct svc_rqst *, struct svc_fh *, struct kstatfs *, int access); -__be32 nfsd_permission(struct svc_rqst *, struct svc_export *, - struct dentry *, int); +__be32 nfsd_permission(struct svc_cred *cred, struct svc_export *exp, + struct dentry *dentry, int acc); void nfsd_filp_close(struct file *fp); From 4f67d24f723b27ed50962db32718d180d0e869da Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Fri, 26 Jul 2024 12:21:33 +1000 Subject: [PATCH 227/302] nfsd: use nfsd_v4client() in nfsd_breaker_owns_lease() nfsd_breaker_owns_lease() currently open-codes the same test that nfsd_v4client() performs. With this patch we use nfsd_v4client() instead. Also as i_am_nfsd() is only used in combination with kthread_data(), replace it with nfsd_current_rqst() which combines the two and returns a valid svc_rqst, or NULL. The test for NULL is moved into nfsd_v4client() for code clarity. Signed-off-by: NeilBrown Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/nfs4state.c | 7 ++----- fs/nfsd/nfsd.h | 4 ++-- fs/nfsd/nfssvc.c | 6 ++++-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index e18b7ef14f59..1109e79351a8 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -5277,11 +5277,8 @@ static bool nfsd_breaker_owns_lease(struct file_lease *fl) struct svc_rqst *rqst; struct nfs4_client *clp; - if (!i_am_nfsd()) - return false; - rqst = kthread_data(current); - /* Note rq_prog == NFS_ACL_PROGRAM is also possible: */ - if (rqst->rq_prog != NFS_PROGRAM || rqst->rq_vers < 4) + rqst = nfsd_current_rqst(); + if (!nfsd_v4client(rqst)) return false; clp = *(rqst->rq_lease_breaker); return dl->dl_stid.sc_client == clp; diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h index 369c3b3ce53e..8750ade0a7ee 100644 --- a/fs/nfsd/nfsd.h +++ b/fs/nfsd/nfsd.h @@ -118,7 +118,7 @@ int nfsd_get_nrthreads(int n, int *, struct net *); int nfsd_set_nrthreads(int n, int *, struct net *); void nfsd_shutdown_threads(struct net *net); -bool i_am_nfsd(void); +struct svc_rqst *nfsd_current_rqst(void); struct nfsdfs_client { struct kref cl_ref; @@ -159,7 +159,7 @@ extern int nfsd_max_blksize; static inline int nfsd_v4client(struct svc_rqst *rq) { - return rq->rq_prog == NFS_PROGRAM && rq->rq_vers == 4; + return rq && rq->rq_prog == NFS_PROGRAM && rq->rq_vers == 4; } static inline struct user_namespace * nfsd_user_namespace(const struct svc_rqst *rqstp) diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c index 3fb6c8c9a2f0..defc430f912f 100644 --- a/fs/nfsd/nfssvc.c +++ b/fs/nfsd/nfssvc.c @@ -578,9 +578,11 @@ void nfsd_shutdown_threads(struct net *net) mutex_unlock(&nfsd_mutex); } -bool i_am_nfsd(void) +struct svc_rqst *nfsd_current_rqst(void) { - return kthread_func(current) == nfsd; + if (kthread_func(current) == nfsd) + return kthread_data(current); + return NULL; } int nfsd_create_serv(struct net *net) From c689bdd3bffad887207bc5b36abdc0efeb2e54d5 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Fri, 26 Jul 2024 12:21:34 +1000 Subject: [PATCH 228/302] nfsd: further centralize protocol version checks. With this patch the only places that test ->rq_vers against a specific version are nfsd_v4client() and nfsd_set_fh_dentry(). The latter sets some flags in the svc_fh, which now includes: fh_64bit_cookies fh_use_wgather Signed-off-by: NeilBrown Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/nfsfh.c | 4 ++++ fs/nfsd/nfsfh.h | 2 ++ fs/nfsd/vfs.c | 9 +++------ 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c index fffeba082540..7abbd8b4a533 100644 --- a/fs/nfsd/nfsfh.c +++ b/fs/nfsd/nfsfh.c @@ -284,13 +284,17 @@ static __be32 nfsd_set_fh_dentry(struct svc_rqst *rqstp, struct svc_fh *fhp) case 4: if (dentry->d_sb->s_export_op->flags & EXPORT_OP_NOATOMIC_ATTR) fhp->fh_no_atomic_attr = true; + fhp->fh_64bit_cookies = true; break; case 3: if (dentry->d_sb->s_export_op->flags & EXPORT_OP_NOWCC) fhp->fh_no_wcc = true; + fhp->fh_64bit_cookies = true; break; case 2: fhp->fh_no_wcc = true; + if (EX_WGATHER(exp)) + fhp->fh_use_wgather = true; } return 0; diff --git a/fs/nfsd/nfsfh.h b/fs/nfsd/nfsfh.h index 6ebdf7ea27bf..8d46e203d139 100644 --- a/fs/nfsd/nfsfh.h +++ b/fs/nfsd/nfsfh.h @@ -88,6 +88,8 @@ typedef struct svc_fh { * wcc data is not atomic with * operation */ + bool fh_use_wgather; /* NFSv2 wgather option */ + bool fh_64bit_cookies;/* readdir cookie size */ int fh_flags; /* FH flags */ bool fh_post_saved; /* post-op attrs saved */ bool fh_pre_saved; /* pre-op attrs saved */ diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c index 0862f6ae86a9..bd6017510109 100644 --- a/fs/nfsd/vfs.c +++ b/fs/nfsd/vfs.c @@ -1162,7 +1162,6 @@ nfsd_vfs_write(struct svc_rqst *rqstp, struct svc_fh *fhp, struct nfsd_file *nf, errseq_t since; __be32 nfserr; int host_err; - int use_wgather; loff_t pos = offset; unsigned long exp_op_flags = 0; unsigned int pflags = current->flags; @@ -1188,12 +1187,11 @@ nfsd_vfs_write(struct svc_rqst *rqstp, struct svc_fh *fhp, struct nfsd_file *nf, } exp = fhp->fh_export; - use_wgather = (rqstp->rq_vers == 2) && EX_WGATHER(exp); if (!EX_ISSYNC(exp)) stable = NFS_UNSTABLE; - if (stable && !use_wgather) + if (stable && !fhp->fh_use_wgather) flags |= RWF_SYNC; iov_iter_kvec(&iter, ITER_SOURCE, vec, vlen, *cnt); @@ -1212,7 +1210,7 @@ nfsd_vfs_write(struct svc_rqst *rqstp, struct svc_fh *fhp, struct nfsd_file *nf, if (host_err < 0) goto out_nfserr; - if (stable && use_wgather) { + if (stable && fhp->fh_use_wgather) { host_err = wait_for_concurrent_writes(file); if (host_err < 0) commit_reset_write_verifier(nn, rqstp, host_err); @@ -2181,8 +2179,7 @@ nfsd_readdir(struct svc_rqst *rqstp, struct svc_fh *fhp, loff_t *offsetp, loff_t offset = *offsetp; int may_flags = NFSD_MAY_READ; - /* NFSv2 only supports 32 bit cookies */ - if (rqstp->rq_vers > 2) + if (fhp->fh_64bit_cookies) may_flags |= NFSD_MAY_64BIT_COOKIE; err = nfsd_open(rqstp, fhp, S_IFDIR, may_flags, &file); From ef7f6c4904d03ccd7478e1ac20ed75f79c4ac444 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Fri, 26 Jul 2024 12:21:35 +1000 Subject: [PATCH 229/302] nfsd: move V4ROOT version check to nfsd_set_fh_dentry() This further centralizes version number checks. Signed-off-by: NeilBrown Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/nfsfh.c | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c index 7abbd8b4a533..d9039383a48c 100644 --- a/fs/nfsd/nfsfh.c +++ b/fs/nfsd/nfsfh.c @@ -116,19 +116,11 @@ static __be32 nfsd_setuser_and_check_port(struct svc_rqst *rqstp, return nfserrno(nfsd_setuser(&rqstp->rq_cred, exp)); } -static inline __be32 check_pseudo_root(struct svc_rqst *rqstp, - struct dentry *dentry, struct svc_export *exp) +static inline __be32 check_pseudo_root(struct dentry *dentry, + struct svc_export *exp) { if (!(exp->ex_flags & NFSEXP_V4ROOT)) return nfs_ok; - /* - * v2/v3 clients have no need for the V4ROOT export--they use - * the mount protocl instead; also, further V4ROOT checks may be - * in v4-specific code, in which case v2/v3 clients could bypass - * them. - */ - if (!nfsd_v4client(rqstp)) - return nfserr_stale; /* * We're exposing only the directories and symlinks that have to be * traversed on the way to real exports: @@ -290,11 +282,15 @@ static __be32 nfsd_set_fh_dentry(struct svc_rqst *rqstp, struct svc_fh *fhp) if (dentry->d_sb->s_export_op->flags & EXPORT_OP_NOWCC) fhp->fh_no_wcc = true; fhp->fh_64bit_cookies = true; + if (exp->ex_flags & NFSEXP_V4ROOT) + goto out; break; case 2: fhp->fh_no_wcc = true; if (EX_WGATHER(exp)) fhp->fh_use_wgather = true; + if (exp->ex_flags & NFSEXP_V4ROOT) + goto out; } return 0; @@ -364,7 +360,7 @@ fh_verify(struct svc_rqst *rqstp, struct svc_fh *fhp, umode_t type, int access) * (for example, if different id-squashing options are in * effect on the new filesystem). */ - error = check_pseudo_root(rqstp, dentry, exp); + error = check_pseudo_root(dentry, exp); if (error) goto out; From 1459ad57673b8019fbfcfddc345c99630d29e716 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Mon, 29 Jul 2024 11:47:22 +1000 Subject: [PATCH 230/302] nfsd: Move error code mapping to per-version proc code. There is code scattered around nfsd which chooses an error status based on the particular version of nfs being used. It is cleaner to have the version specific choices in version specific code. With this patch common code returns the most specific error code possible and the version specific code maps that if necessary. Both v2 (nfsproc.c) and v3 (nfs3proc.c) now have a "map_status()" function which is called to map the resp->status before each non-trivial nfsd_proc_* or nfsd3_proc_* function returns. NFS4ERR_SYMLINK and NFS4ERR_WRONG_TYPE introduce extra complications and are left for a later patch. Signed-off-by: NeilBrown Reviewed-by: Jeff Layton Reviewed-by: Christoph Hellwig Signed-off-by: Chuck Lever --- fs/nfsd/export.c | 2 +- fs/nfsd/nfs3proc.c | 37 +++++++++++++++++++++++++++++++++++++ fs/nfsd/nfsfh.c | 10 +++------- fs/nfsd/nfsproc.c | 33 +++++++++++++++++++++++++++++++++ fs/nfsd/vfs.c | 14 ++++---------- 5 files changed, 78 insertions(+), 18 deletions(-) diff --git a/fs/nfsd/export.c b/fs/nfsd/export.c index 9aa5f95f18a8..7bb4f2075ac5 100644 --- a/fs/nfsd/export.c +++ b/fs/nfsd/export.c @@ -1121,7 +1121,7 @@ ok: return 0; denied: - return rqstp->rq_vers < 4 ? nfserr_acces : nfserr_wrongsec; + return nfserr_wrongsec; } /* diff --git a/fs/nfsd/nfs3proc.c b/fs/nfsd/nfs3proc.c index dfcc957e460d..7581d64edfa3 100644 --- a/fs/nfsd/nfs3proc.c +++ b/fs/nfsd/nfs3proc.c @@ -28,6 +28,22 @@ static int nfs3_ftypes[] = { S_IFIFO, /* NF3FIFO */ }; +static __be32 nfsd3_map_status(__be32 status) +{ + switch (status) { + case nfs_ok: + break; + case nfserr_nofilehandle: + status = nfserr_badhandle; + break; + case nfserr_wrongsec: + case nfserr_file_open: + status = nfserr_acces; + break; + } + return status; +} + /* * NULL call. */ @@ -57,6 +73,7 @@ nfsd3_proc_getattr(struct svc_rqst *rqstp) resp->status = fh_getattr(&resp->fh, &resp->stat); out: + resp->status = nfsd3_map_status(resp->status); return rpc_success; } @@ -80,6 +97,7 @@ nfsd3_proc_setattr(struct svc_rqst *rqstp) if (argp->check_guard) guardtime = &argp->guardtime; resp->status = nfsd_setattr(rqstp, &resp->fh, &attrs, guardtime); + resp->status = nfsd3_map_status(resp->status); return rpc_success; } @@ -103,6 +121,7 @@ nfsd3_proc_lookup(struct svc_rqst *rqstp) resp->status = nfsd_lookup(rqstp, &resp->dirfh, argp->name, argp->len, &resp->fh); + resp->status = nfsd3_map_status(resp->status); return rpc_success; } @@ -122,6 +141,7 @@ nfsd3_proc_access(struct svc_rqst *rqstp) fh_copy(&resp->fh, &argp->fh); resp->access = argp->access; resp->status = nfsd_access(rqstp, &resp->fh, &resp->access, NULL); + resp->status = nfsd3_map_status(resp->status); return rpc_success; } @@ -142,6 +162,7 @@ nfsd3_proc_readlink(struct svc_rqst *rqstp) resp->pages = rqstp->rq_next_page++; resp->status = nfsd_readlink(rqstp, &resp->fh, page_address(*resp->pages), &resp->len); + resp->status = nfsd3_map_status(resp->status); return rpc_success; } @@ -179,6 +200,7 @@ nfsd3_proc_read(struct svc_rqst *rqstp) fh_copy(&resp->fh, &argp->fh); resp->status = nfsd_read(rqstp, &resp->fh, argp->offset, &resp->count, &resp->eof); + resp->status = nfsd3_map_status(resp->status); return rpc_success; } @@ -212,6 +234,7 @@ nfsd3_proc_write(struct svc_rqst *rqstp) rqstp->rq_vec, nvecs, &cnt, resp->committed, resp->verf); resp->count = cnt; + resp->status = nfsd3_map_status(resp->status); return rpc_success; } @@ -359,6 +382,7 @@ nfsd3_proc_create(struct svc_rqst *rqstp) newfhp = fh_init(&resp->fh, NFS3_FHSIZE); resp->status = nfsd3_create_file(rqstp, dirfhp, newfhp, argp); + resp->status = nfsd3_map_status(resp->status); return rpc_success; } @@ -384,6 +408,7 @@ nfsd3_proc_mkdir(struct svc_rqst *rqstp) fh_init(&resp->fh, NFS3_FHSIZE); resp->status = nfsd_create(rqstp, &resp->dirfh, argp->name, argp->len, &attrs, S_IFDIR, 0, &resp->fh); + resp->status = nfsd3_map_status(resp->status); return rpc_success; } @@ -424,6 +449,7 @@ nfsd3_proc_symlink(struct svc_rqst *rqstp) argp->flen, argp->tname, &attrs, &resp->fh); kfree(argp->tname); out: + resp->status = nfsd3_map_status(resp->status); return rpc_success; } @@ -465,6 +491,7 @@ nfsd3_proc_mknod(struct svc_rqst *rqstp) resp->status = nfsd_create(rqstp, &resp->dirfh, argp->name, argp->len, &attrs, type, rdev, &resp->fh); out: + resp->status = nfsd3_map_status(resp->status); return rpc_success; } @@ -486,6 +513,7 @@ nfsd3_proc_remove(struct svc_rqst *rqstp) fh_copy(&resp->fh, &argp->fh); resp->status = nfsd_unlink(rqstp, &resp->fh, -S_IFDIR, argp->name, argp->len); + resp->status = nfsd3_map_status(resp->status); return rpc_success; } @@ -506,6 +534,7 @@ nfsd3_proc_rmdir(struct svc_rqst *rqstp) fh_copy(&resp->fh, &argp->fh); resp->status = nfsd_unlink(rqstp, &resp->fh, S_IFDIR, argp->name, argp->len); + resp->status = nfsd3_map_status(resp->status); return rpc_success; } @@ -528,6 +557,7 @@ nfsd3_proc_rename(struct svc_rqst *rqstp) fh_copy(&resp->tfh, &argp->tfh); resp->status = nfsd_rename(rqstp, &resp->ffh, argp->fname, argp->flen, &resp->tfh, argp->tname, argp->tlen); + resp->status = nfsd3_map_status(resp->status); return rpc_success; } @@ -548,6 +578,7 @@ nfsd3_proc_link(struct svc_rqst *rqstp) fh_copy(&resp->tfh, &argp->tfh); resp->status = nfsd_link(rqstp, &resp->tfh, argp->tname, argp->tlen, &resp->fh); + resp->status = nfsd3_map_status(resp->status); return rpc_success; } @@ -600,6 +631,7 @@ nfsd3_proc_readdir(struct svc_rqst *rqstp) /* Recycle only pages that were part of the reply */ rqstp->rq_next_page = resp->xdr.page_ptr + 1; + resp->status = nfsd3_map_status(resp->status); return rpc_success; } @@ -644,6 +676,7 @@ nfsd3_proc_readdirplus(struct svc_rqst *rqstp) rqstp->rq_next_page = resp->xdr.page_ptr + 1; out: + resp->status = nfsd3_map_status(resp->status); return rpc_success; } @@ -661,6 +694,7 @@ nfsd3_proc_fsstat(struct svc_rqst *rqstp) resp->status = nfsd_statfs(rqstp, &argp->fh, &resp->stats, 0); fh_put(&argp->fh); + resp->status = nfsd3_map_status(resp->status); return rpc_success; } @@ -704,6 +738,7 @@ nfsd3_proc_fsinfo(struct svc_rqst *rqstp) } fh_put(&argp->fh); + resp->status = nfsd3_map_status(resp->status); return rpc_success; } @@ -746,6 +781,7 @@ nfsd3_proc_pathconf(struct svc_rqst *rqstp) } fh_put(&argp->fh); + resp->status = nfsd3_map_status(resp->status); return rpc_success; } @@ -773,6 +809,7 @@ nfsd3_proc_commit(struct svc_rqst *rqstp) argp->count, resp->verf); nfsd_file_put(nf); out: + resp->status = nfsd3_map_status(resp->status); return rpc_success; } diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c index d9039383a48c..388c2952a0a8 100644 --- a/fs/nfsd/nfsfh.c +++ b/fs/nfsd/nfsfh.c @@ -154,10 +154,8 @@ static __be32 nfsd_set_fh_dentry(struct svc_rqst *rqstp, struct svc_fh *fhp) int len; __be32 error; - error = nfserr_stale; - if (rqstp->rq_vers > 2) - error = nfserr_badhandle; - if (rqstp->rq_vers == 4 && fh->fh_size == 0) + error = nfserr_badhandle; + if (fh->fh_size == 0) return nfserr_nofilehandle; if (fh->fh_version != 1) @@ -231,9 +229,7 @@ static __be32 nfsd_set_fh_dentry(struct svc_rqst *rqstp, struct svc_fh *fhp) /* * Look up the dentry using the NFS file handle. */ - error = nfserr_stale; - if (rqstp->rq_vers > 2) - error = nfserr_badhandle; + error = nfserr_badhandle; fileid_type = fh->fh_fileid_type; diff --git a/fs/nfsd/nfsproc.c b/fs/nfsd/nfsproc.c index 97aab34593ef..5392f4775ee8 100644 --- a/fs/nfsd/nfsproc.c +++ b/fs/nfsd/nfsproc.c @@ -13,6 +13,24 @@ #define NFSDDBG_FACILITY NFSDDBG_PROC +static __be32 nfsd_map_status(__be32 status) +{ + switch (status) { + case nfs_ok: + break; + case nfserr_nofilehandle: + case nfserr_badhandle: + status = nfserr_stale; + break; + case nfserr_wrongsec: + case nfserr_xdev: + case nfserr_file_open: + status = nfserr_acces; + break; + } + return status; +} + static __be32 nfsd_proc_null(struct svc_rqst *rqstp) { @@ -38,6 +56,7 @@ nfsd_proc_getattr(struct svc_rqst *rqstp) goto out; resp->status = fh_getattr(&resp->fh, &resp->stat); out: + resp->status = nfsd_map_status(resp->status); return rpc_success; } @@ -109,6 +128,7 @@ nfsd_proc_setattr(struct svc_rqst *rqstp) resp->status = fh_getattr(&resp->fh, &resp->stat); out: + resp->status = nfsd_map_status(resp->status); return rpc_success; } @@ -143,6 +163,7 @@ nfsd_proc_lookup(struct svc_rqst *rqstp) resp->status = fh_getattr(&resp->fh, &resp->stat); out: + resp->status = nfsd_map_status(resp->status); return rpc_success; } @@ -164,6 +185,7 @@ nfsd_proc_readlink(struct svc_rqst *rqstp) page_address(resp->page), &resp->len); fh_put(&argp->fh); + resp->status = nfsd_map_status(resp->status); return rpc_success; } @@ -200,6 +222,7 @@ nfsd_proc_read(struct svc_rqst *rqstp) resp->status = fh_getattr(&resp->fh, &resp->stat); else if (resp->status == nfserr_jukebox) set_bit(RQ_DROPME, &rqstp->rq_flags); + resp->status = nfsd_map_status(resp->status); return rpc_success; } @@ -235,6 +258,7 @@ nfsd_proc_write(struct svc_rqst *rqstp) resp->status = fh_getattr(&resp->fh, &resp->stat); else if (resp->status == nfserr_jukebox) set_bit(RQ_DROPME, &rqstp->rq_flags); + resp->status = nfsd_map_status(resp->status); return rpc_success; } @@ -404,6 +428,7 @@ done: goto out; resp->status = fh_getattr(&resp->fh, &resp->stat); out: + resp->status = nfsd_map_status(resp->status); return rpc_success; } @@ -420,6 +445,7 @@ nfsd_proc_remove(struct svc_rqst *rqstp) resp->status = nfsd_unlink(rqstp, &argp->fh, -S_IFDIR, argp->name, argp->len); fh_put(&argp->fh); + resp->status = nfsd_map_status(resp->status); return rpc_success; } @@ -438,6 +464,7 @@ nfsd_proc_rename(struct svc_rqst *rqstp) &argp->tfh, argp->tname, argp->tlen); fh_put(&argp->ffh); fh_put(&argp->tfh); + resp->status = nfsd_map_status(resp->status); return rpc_success; } @@ -458,6 +485,7 @@ nfsd_proc_link(struct svc_rqst *rqstp) &argp->ffh); fh_put(&argp->ffh); fh_put(&argp->tfh); + resp->status = nfsd_map_status(resp->status); return rpc_success; } @@ -496,6 +524,7 @@ nfsd_proc_symlink(struct svc_rqst *rqstp) fh_put(&argp->ffh); fh_put(&newfh); out: + resp->status = nfsd_map_status(resp->status); return rpc_success; } @@ -529,6 +558,7 @@ nfsd_proc_mkdir(struct svc_rqst *rqstp) resp->status = fh_getattr(&resp->fh, &resp->stat); out: + resp->status = nfsd_map_status(resp->status); return rpc_success; } @@ -546,6 +576,7 @@ nfsd_proc_rmdir(struct svc_rqst *rqstp) resp->status = nfsd_unlink(rqstp, &argp->fh, S_IFDIR, argp->name, argp->len); fh_put(&argp->fh); + resp->status = nfsd_map_status(resp->status); return rpc_success; } @@ -591,6 +622,7 @@ nfsd_proc_readdir(struct svc_rqst *rqstp) nfssvc_encode_nfscookie(resp, offset); fh_put(&argp->fh); + resp->status = nfsd_map_status(resp->status); return rpc_success; } @@ -608,6 +640,7 @@ nfsd_proc_statfs(struct svc_rqst *rqstp) resp->status = nfsd_statfs(rqstp, &argp->fh, &resp->stats, NFSD_MAY_BYPASS_GSS_ON_ROOT); fh_put(&argp->fh); + resp->status = nfsd_map_status(resp->status); return rpc_success; } diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c index bd6017510109..ec99c91df173 100644 --- a/fs/nfsd/vfs.c +++ b/fs/nfsd/vfs.c @@ -1768,10 +1768,7 @@ nfsd_link(struct svc_rqst *rqstp, struct svc_fh *ffhp, if (!err) err = nfserrno(commit_metadata(tfhp)); } else { - if (host_err == -EXDEV && rqstp->rq_vers == 2) - err = nfserr_acces; - else - err = nfserrno(host_err); + err = nfserrno(host_err); } dput(dnew); out_drop_write: @@ -1837,7 +1834,7 @@ nfsd_rename(struct svc_rqst *rqstp, struct svc_fh *ffhp, char *fname, int flen, if (!flen || isdotent(fname, flen) || !tlen || isdotent(tname, tlen)) goto out; - err = (rqstp->rq_vers == 2) ? nfserr_acces : nfserr_xdev; + err = nfserr_xdev; if (ffhp->fh_export->ex_path.mnt != tfhp->fh_export->ex_path.mnt) goto out; if (ffhp->fh_export->ex_path.dentry != tfhp->fh_export->ex_path.dentry) @@ -1852,7 +1849,7 @@ retry: trap = lock_rename(tdentry, fdentry); if (IS_ERR(trap)) { - err = (rqstp->rq_vers == 2) ? nfserr_acces : nfserr_xdev; + err = nfserr_xdev; goto out_want_write; } err = fh_fill_pre_attrs(ffhp); @@ -2021,10 +2018,7 @@ out_nfserr: /* name is mounted-on. There is no perfect * error status. */ - if (nfsd_v4client(rqstp)) - err = nfserr_file_open; - else - err = nfserr_acces; + err = nfserr_file_open; } else { err = nfserrno(host_err); } From 36ffa3d0de54c1cf516ea32a5ec556f5c9874795 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Mon, 29 Jul 2024 11:47:23 +1000 Subject: [PATCH 231/302] nfsd: be more systematic about selecting error codes for internal use. Rather than using ad hoc values for internal errors (30000, 11000, ...) use 'enum' to sequentially allocate numbers starting from the first known available number - now visible as NFS4ERR_FIRST_FREE. The goal is values that are distinct from all be32 error codes. To get those we must first select integers that are not already used, then convert them with cpu_to_be32(). Signed-off-by: NeilBrown Reviewed-by: Jeff Layton Reviewed-by: Christoph Hellwig Signed-off-by: Chuck Lever --- fs/nfsd/nfsd.h | 24 +++++++++++++++++++----- include/linux/nfs4.h | 17 ++++++++++------- 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h index 8750ade0a7ee..11825cbe4360 100644 --- a/fs/nfsd/nfsd.h +++ b/fs/nfsd/nfsd.h @@ -330,17 +330,31 @@ void nfsd_lockd_shutdown(void); #define nfserr_xattr2big cpu_to_be32(NFS4ERR_XATTR2BIG) #define nfserr_noxattr cpu_to_be32(NFS4ERR_NOXATTR) -/* error codes for internal use */ +/* + * Error codes for internal use. We use enum to choose numbers that are + * not already assigned, then covert to be32 resulting in a number that + * cannot conflict with any existing be32 nfserr value. + */ +enum { + NFSERR_DROPIT = NFS4ERR_FIRST_FREE, /* if a request fails due to kmalloc failure, it gets dropped. * Client should resend eventually */ -#define nfserr_dropit cpu_to_be32(30000) +#define nfserr_dropit cpu_to_be32(NFSERR_DROPIT) + /* end-of-file indicator in readdir */ -#define nfserr_eof cpu_to_be32(30001) + NFSERR_EOF, +#define nfserr_eof cpu_to_be32(NFSERR_EOF) + /* replay detected */ -#define nfserr_replay_me cpu_to_be32(11001) + NFSERR_REPLAY_ME, +#define nfserr_replay_me cpu_to_be32(NFSERR_REPLAY_ME) + /* nfs41 replay detected */ -#define nfserr_replay_cache cpu_to_be32(11002) + NFSERR_REPLAY_CACHE, +#define nfserr_replay_cache cpu_to_be32(NFSERR_REPLAY_CACHE) + +}; /* Check for dir entries '.' and '..' */ #define isdotent(n, l) (l < 3 && n[0] == '.' && (l == 1 || n[1] == '.')) diff --git a/include/linux/nfs4.h b/include/linux/nfs4.h index f9df88091c6d..8d7430d9f218 100644 --- a/include/linux/nfs4.h +++ b/include/linux/nfs4.h @@ -281,15 +281,18 @@ enum nfsstat4 { /* nfs42 */ NFS4ERR_PARTNER_NOTSUPP = 10088, NFS4ERR_PARTNER_NO_AUTH = 10089, - NFS4ERR_UNION_NOTSUPP = 10090, - NFS4ERR_OFFLOAD_DENIED = 10091, - NFS4ERR_WRONG_LFS = 10092, - NFS4ERR_BADLABEL = 10093, - NFS4ERR_OFFLOAD_NO_REQS = 10094, + NFS4ERR_UNION_NOTSUPP = 10090, + NFS4ERR_OFFLOAD_DENIED = 10091, + NFS4ERR_WRONG_LFS = 10092, + NFS4ERR_BADLABEL = 10093, + NFS4ERR_OFFLOAD_NO_REQS = 10094, /* xattr (RFC8276) */ - NFS4ERR_NOXATTR = 10095, - NFS4ERR_XATTR2BIG = 10096, + NFS4ERR_NOXATTR = 10095, + NFS4ERR_XATTR2BIG = 10096, + + /* can be used for internal errors */ + NFS4ERR_FIRST_FREE }; /* error codes for internal client use */ From 438f81e0e92a780b117097503599eb030b77dabe Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Wed, 14 Aug 2024 09:21:01 -0400 Subject: [PATCH 232/302] nfsd: move error choice for incorrect object types to version-specific code. If an NFS operation expects a particular sort of object (file, dir, link, etc) but gets a file handle for a different sort of object, it must return an error. The actual error varies among NFS versions in non-trivial ways. For v2 and v3 there are ISDIR and NOTDIR errors and, for NFSv4 only, INVAL is suitable. For v4.0 there is also NFS4ERR_SYMLINK which should be used if a SYMLINK was found when not expected. This take precedence over NOTDIR. For v4.1+ there is also NFS4ERR_WRONG_TYPE which should be used in preference to EINVAL when none of the specific error codes apply. When nfsd_mode_check() finds a symlink where it expected a directory it needs to return an error code that can be converted to NOTDIR for v2 or v3 but will be SYMLINK for v4. It must be different from the error code returns when it finds a symlink but expects a regular file - that must be converted to EINVAL or SYMLINK. So we introduce an internal error code nfserr_symlink_not_dir which each version converts as appropriate. nfsd_check_obj_isreg() is similar to nfsd_mode_check() except that it is only used by NFSv4 and only for OPEN. NFSERR_INVAL is never a suitable error if the object is the wrong time. For v4.0 we use nfserr_symlink for non-dirs even if not a symlink. For v4.1 we have nfserr_wrong_type. We handle this difference in-place in nfsd_check_obj_isreg() as there is nothing to be gained by delaying the choice to nfsd4_map_status(). As a result of these changes, nfsd_mode_check() doesn't need an rqstp arg any more. Note that NFSv4 operations are actually performed in the xdr code(!!!) so to the only place that we can map the status code successfully is in nfsd4_encode_operation(). Signed-off-by: NeilBrown Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/nfs3proc.c | 7 +++++++ fs/nfsd/nfs4proc.c | 21 +++++++++++---------- fs/nfsd/nfs4xdr.c | 19 +++++++++++++++++++ fs/nfsd/nfsd.h | 5 +++++ fs/nfsd/nfsfh.c | 16 +++++++--------- fs/nfsd/nfsproc.c | 7 +++++++ 6 files changed, 56 insertions(+), 19 deletions(-) diff --git a/fs/nfsd/nfs3proc.c b/fs/nfsd/nfs3proc.c index 7581d64edfa3..372bdcf5e07a 100644 --- a/fs/nfsd/nfs3proc.c +++ b/fs/nfsd/nfs3proc.c @@ -40,6 +40,13 @@ static __be32 nfsd3_map_status(__be32 status) case nfserr_file_open: status = nfserr_acces; break; + case nfserr_symlink_not_dir: + status = nfserr_notdir; + break; + case nfserr_symlink: + case nfserr_wrong_type: + status = nfserr_inval; + break; } return status; } diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index 4f5b8c4be417..fc68af757080 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -158,7 +158,7 @@ do_open_permission(struct svc_rqst *rqstp, struct svc_fh *current_fh, struct nfs return fh_verify(rqstp, current_fh, S_IFREG, accmode); } -static __be32 nfsd_check_obj_isreg(struct svc_fh *fh) +static __be32 nfsd_check_obj_isreg(struct svc_fh *fh, u32 minor_version) { umode_t mode = d_inode(fh->fh_dentry)->i_mode; @@ -166,14 +166,15 @@ static __be32 nfsd_check_obj_isreg(struct svc_fh *fh) return nfs_ok; if (S_ISDIR(mode)) return nfserr_isdir; - /* - * Using err_symlink as our catch-all case may look odd; but - * there's no other obvious error for this case in 4.0, and we - * happen to know that it will cause the linux v4 client to do - * the right thing on attempts to open something other than a - * regular file. - */ - return nfserr_symlink; + if (S_ISLNK(mode)) + return nfserr_symlink; + + /* RFC 7530 - 16.16.6 */ + if (minor_version == 0) + return nfserr_symlink; + else + return nfserr_wrong_type; + } static void nfsd4_set_open_owner_reply_cache(struct nfsd4_compound_state *cstate, struct nfsd4_open *open, struct svc_fh *resfh) @@ -466,7 +467,7 @@ do_open_lookup(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, stru } if (status) goto out; - status = nfsd_check_obj_isreg(*resfh); + status = nfsd_check_obj_isreg(*resfh, cstate->minorversion); if (status) goto out; diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c index 97f583777972..4643fcfb7187 100644 --- a/fs/nfsd/nfs4xdr.c +++ b/fs/nfsd/nfs4xdr.c @@ -5731,6 +5731,23 @@ __be32 nfsd4_check_resp_size(struct nfsd4_compoundres *resp, u32 respsize) return nfserr_rep_too_big; } +static __be32 nfsd4_map_status(__be32 status, u32 minor) +{ + switch (status) { + case nfs_ok: + break; + case nfserr_wrong_type: + /* RFC 8881 - 15.1.2.9 */ + if (minor == 0) + status = nfserr_inval; + break; + case nfserr_symlink_not_dir: + status = nfserr_symlink; + break; + } + return status; +} + void nfsd4_encode_operation(struct nfsd4_compoundres *resp, struct nfsd4_op *op) { @@ -5798,6 +5815,8 @@ nfsd4_encode_operation(struct nfsd4_compoundres *resp, struct nfsd4_op *op) so->so_replay.rp_buf, len); } status: + op->status = nfsd4_map_status(op->status, + resp->cstate.minorversion); *p = op->status; release: if (opdesc && opdesc->op_release) diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h index 11825cbe4360..4ccbf014a2c7 100644 --- a/fs/nfsd/nfsd.h +++ b/fs/nfsd/nfsd.h @@ -354,6 +354,11 @@ enum { NFSERR_REPLAY_CACHE, #define nfserr_replay_cache cpu_to_be32(NFSERR_REPLAY_CACHE) +/* symlink found where dir expected - handled differently to + * other symlink found errors by NFSv3. + */ + NFSERR_SYMLINK_NOT_DIR, +#define nfserr_symlink_not_dir cpu_to_be32(NFSERR_SYMLINK_NOT_DIR) }; /* Check for dir entries '.' and '..' */ diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c index 388c2952a0a8..50d23d56f403 100644 --- a/fs/nfsd/nfsfh.c +++ b/fs/nfsd/nfsfh.c @@ -62,8 +62,7 @@ static int nfsd_acceptable(void *expv, struct dentry *dentry) * the write call). */ static inline __be32 -nfsd_mode_check(struct svc_rqst *rqstp, struct dentry *dentry, - umode_t requested) +nfsd_mode_check(struct dentry *dentry, umode_t requested) { umode_t mode = d_inode(dentry)->i_mode & S_IFMT; @@ -76,17 +75,16 @@ nfsd_mode_check(struct svc_rqst *rqstp, struct dentry *dentry, } return nfs_ok; } - /* - * v4 has an error more specific than err_notdir which we should - * return in preference to err_notdir: - */ - if (rqstp->rq_vers == 4 && mode == S_IFLNK) + if (mode == S_IFLNK) { + if (requested == S_IFDIR) + return nfserr_symlink_not_dir; return nfserr_symlink; + } if (requested == S_IFDIR) return nfserr_notdir; if (mode == S_IFDIR) return nfserr_isdir; - return nfserr_inval; + return nfserr_wrong_type; } static bool nfsd_originating_port_ok(struct svc_rqst *rqstp, int flags) @@ -364,7 +362,7 @@ fh_verify(struct svc_rqst *rqstp, struct svc_fh *fhp, umode_t type, int access) if (error) goto out; - error = nfsd_mode_check(rqstp, dentry, type); + error = nfsd_mode_check(dentry, type); if (error) goto out; diff --git a/fs/nfsd/nfsproc.c b/fs/nfsd/nfsproc.c index 5392f4775ee8..6dda081eb24c 100644 --- a/fs/nfsd/nfsproc.c +++ b/fs/nfsd/nfsproc.c @@ -27,6 +27,13 @@ static __be32 nfsd_map_status(__be32 status) case nfserr_file_open: status = nfserr_acces; break; + case nfserr_symlink_not_dir: + status = nfserr_notdir; + break; + case nfserr_symlink: + case nfserr_wrong_type: + status = nfserr_inval; + break; } return status; } From c4de97f7c45434985e5dbf2d6ccc9eca676e37fe Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Mon, 29 Jul 2024 16:52:32 -0400 Subject: [PATCH 233/302] svcrdma: Handle device removal outside of the CM event handler Synchronously wait for all disconnects to complete to ensure the transports have divested all hardware resources before the underlying RDMA device can safely be removed. Reviewed-by: Sagi Grimberg Signed-off-by: Chuck Lever --- include/linux/sunrpc/svc_rdma.h | 2 ++ include/trace/events/rpcrdma.h | 23 +++++++++++++++++++++++ net/sunrpc/xprtrdma/svc_rdma_transport.c | 16 +++++++++++++++- 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h index d33bab33099a..619fc0bd837a 100644 --- a/include/linux/sunrpc/svc_rdma.h +++ b/include/linux/sunrpc/svc_rdma.h @@ -48,6 +48,7 @@ #include #include #include +#include #include #include @@ -76,6 +77,7 @@ struct svcxprt_rdma { struct svc_xprt sc_xprt; /* SVC transport structure */ struct rdma_cm_id *sc_cm_id; /* RDMA connection id */ struct list_head sc_accept_q; /* Conn. waiting accept */ + struct rpcrdma_notification sc_rn; /* removal notification */ int sc_ord; /* RDMA read limit */ int sc_max_send_sges; bool sc_snd_w_inv; /* OK to use Send With Invalidate */ diff --git a/include/trace/events/rpcrdma.h b/include/trace/events/rpcrdma.h index a96a985c49b3..e6a72646c507 100644 --- a/include/trace/events/rpcrdma.h +++ b/include/trace/events/rpcrdma.h @@ -2172,6 +2172,29 @@ TRACE_EVENT(svcrdma_qp_error, ) ); +TRACE_EVENT(svcrdma_device_removal, + TP_PROTO( + const struct rdma_cm_id *id + ), + + TP_ARGS(id), + + TP_STRUCT__entry( + __string(name, id->device->name) + __array(unsigned char, addr, sizeof(struct sockaddr_in6)) + ), + + TP_fast_assign( + __assign_str(name); + memcpy(__entry->addr, &id->route.addr.dst_addr, + sizeof(struct sockaddr_in6)); + ), + + TP_printk("device %s to be removed, disconnecting %pISpc\n", + __get_str(name), __entry->addr + ) +); + DECLARE_EVENT_CLASS(svcrdma_sendqueue_class, TP_PROTO( const struct svcxprt_rdma *rdma, diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c index f15750cacacf..581cc5ed7c0c 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_transport.c +++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c @@ -339,7 +339,6 @@ static int svc_rdma_cma_handler(struct rdma_cm_id *cma_id, svc_xprt_enqueue(xprt); break; case RDMA_CM_EVENT_DISCONNECTED: - case RDMA_CM_EVENT_DEVICE_REMOVAL: svc_xprt_deferred_close(xprt); break; default: @@ -384,6 +383,16 @@ static struct svc_xprt *svc_rdma_create(struct svc_serv *serv, return &cma_xprt->sc_xprt; } +static void svc_rdma_xprt_done(struct rpcrdma_notification *rn) +{ + struct svcxprt_rdma *rdma = container_of(rn, struct svcxprt_rdma, + sc_rn); + struct rdma_cm_id *id = rdma->sc_cm_id; + + trace_svcrdma_device_removal(id); + svc_xprt_close(&rdma->sc_xprt); +} + /* * This is the xpo_recvfrom function for listening endpoints. Its * purpose is to accept incoming connections. The CMA callback handler @@ -425,6 +434,9 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) dev = newxprt->sc_cm_id->device; newxprt->sc_port_num = newxprt->sc_cm_id->port_num; + if (rpcrdma_rn_register(dev, &newxprt->sc_rn, svc_rdma_xprt_done)) + goto errout; + newxprt->sc_max_req_size = svcrdma_max_req_size; newxprt->sc_max_requests = svcrdma_max_requests; newxprt->sc_max_bc_requests = svcrdma_max_bc_requests; @@ -580,6 +592,7 @@ static void __svc_rdma_free(struct work_struct *work) { struct svcxprt_rdma *rdma = container_of(work, struct svcxprt_rdma, sc_work); + struct ib_device *device = rdma->sc_cm_id->device; /* This blocks until the Completion Queues are empty */ if (rdma->sc_qp && !IS_ERR(rdma->sc_qp)) @@ -608,6 +621,7 @@ static void __svc_rdma_free(struct work_struct *work) /* Destroy the CM ID */ rdma_destroy_id(rdma->sc_cm_id); + rpcrdma_rn_unregister(device, &rdma->sc_rn); kfree(rdma); } From 32b34fa485645221e4f4b2e7ba8a02260920dce9 Mon Sep 17 00:00:00 2001 From: Mark Grimes Date: Tue, 6 Aug 2024 18:58:34 -0700 Subject: [PATCH 234/302] nfsd: Add quotes to client info 'callback address' The 'callback address' in client_info_show is output without quotes causing yaml parsers to fail on processing IPv6 addresses. Adding quotes to 'callback address' also matches that used by the 'address' field. Signed-off-by: Mark Grimes Reviewed-by: Jeff Layton Reviewed-by: NeilBrown Signed-off-by: Chuck Lever --- fs/nfsd/nfs4state.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 1109e79351a8..6eb61b6e4ac1 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -2692,7 +2692,7 @@ static int client_info_show(struct seq_file *m, void *v) clp->cl_nii_time.tv_sec, clp->cl_nii_time.tv_nsec); } seq_printf(m, "callback state: %s\n", cb_state2str(clp->cl_cb_state)); - seq_printf(m, "callback address: %pISpc\n", &clp->cl_cb_conn.cb_addr); + seq_printf(m, "callback address: \"%pISpc\"\n", &clp->cl_cb_conn.cb_addr); seq_printf(m, "admin-revoked states: %d\n", atomic_read(&clp->cl_admin_revoked)); drop_client(clp); From 202f39039a11402dcbcd5fece8d9fa6be83f49ae Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Sun, 11 Aug 2024 13:11:07 -0400 Subject: [PATCH 235/302] NFSD: Fix NFSv4's PUTPUBFH operation According to RFC 8881, all minor versions of NFSv4 support PUTPUBFH. Replace the XDR decoder for PUTPUBFH with a "noop" since we no longer want the minorversion check, and PUTPUBFH has no arguments to decode. (Ideally nfsd4_decode_noop should really be called nfsd4_decode_void). PUTPUBFH should now behave just like PUTROOTFH. Reported-by: Cedric Blancher Fixes: e1a90ebd8b23 ("NFSD: Combine decode operations for v4 and v4.1") Cc: Dan Shelton Cc: Roland Mainz Cc: stable@vger.kernel.org Signed-off-by: Chuck Lever --- fs/nfsd/nfs4xdr.c | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c index 4643fcfb7187..f118921250c3 100644 --- a/fs/nfsd/nfs4xdr.c +++ b/fs/nfsd/nfs4xdr.c @@ -1245,14 +1245,6 @@ nfsd4_decode_putfh(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u) return nfs_ok; } -static __be32 -nfsd4_decode_putpubfh(struct nfsd4_compoundargs *argp, union nfsd4_op_u *p) -{ - if (argp->minorversion == 0) - return nfs_ok; - return nfserr_notsupp; -} - static __be32 nfsd4_decode_read(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u) { @@ -2374,7 +2366,7 @@ static const nfsd4_dec nfsd4_dec_ops[] = { [OP_OPEN_CONFIRM] = nfsd4_decode_open_confirm, [OP_OPEN_DOWNGRADE] = nfsd4_decode_open_downgrade, [OP_PUTFH] = nfsd4_decode_putfh, - [OP_PUTPUBFH] = nfsd4_decode_putpubfh, + [OP_PUTPUBFH] = nfsd4_decode_noop, [OP_PUTROOTFH] = nfsd4_decode_noop, [OP_READ] = nfsd4_decode_read, [OP_READDIR] = nfsd4_decode_readdir, From ecbf84940575e107df13f308121682f7e8c0d20b Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 13 Aug 2024 17:55:45 -0400 Subject: [PATCH 236/302] .mailmap: Add an entry for my work email address Collect a few very old previous employers as well. Signed-off-by: Chuck Lever --- .mailmap | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.mailmap b/.mailmap index caf46a652f15..40dcf481331b 100644 --- a/.mailmap +++ b/.mailmap @@ -153,6 +153,9 @@ Christian Brauner Christian Marangi Christophe Ricard Christoph Hellwig +Chuck Lever +Chuck Lever +Chuck Lever Claudiu Beznea Colin Ian King Corey Minyard From 2039c5da5dc110ba56aa811f1688c184eed5d2f5 Mon Sep 17 00:00:00 2001 From: Li Lingfeng Date: Wed, 14 Aug 2024 19:29:07 +0800 Subject: [PATCH 237/302] NFSD: remove redundant assignment operation Commit 5826e09bf3dd ("NFSD: OP_CB_RECALL_ANY should recall both read and write delegations") added a new assignment statement to add RCA4_TYPE_MASK_WDATA_DLG to ra_bmval bitmask of OP_CB_RECALL_ANY. So the old one should be removed. Signed-off-by: Li Lingfeng Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/nfs4state.c | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 6eb61b6e4ac1..fa58de23a753 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -6643,7 +6643,6 @@ deleg_reaper(struct nfsd_net *nn) cl_ra_cblist); list_del_init(&clp->cl_ra_cblist); clp->cl_ra->ra_keep = 0; - clp->cl_ra->ra_bmval[0] = BIT(RCA4_TYPE_MASK_RDATA_DLG); clp->cl_ra->ra_bmval[0] = BIT(RCA4_TYPE_MASK_RDATA_DLG) | BIT(RCA4_TYPE_MASK_WDATA_DLG); trace_nfsd_cb_recall_any(clp->cl_ra); From 340e61e44c1d2a15c42ec72ade9195ad525fd048 Mon Sep 17 00:00:00 2001 From: Li Lingfeng Date: Sat, 17 Aug 2024 14:27:13 +0800 Subject: [PATCH 238/302] nfsd: map the EBADMSG to nfserr_io to avoid warning Ext4 will throw -EBADMSG through ext4_readdir when a checksum error occurs, resulting in the following WARNING. Fix it by mapping EBADMSG to nfserr_io. nfsd_buffered_readdir iterate_dir // -EBADMSG -74 ext4_readdir // .iterate_shared ext4_dx_readdir ext4_htree_fill_tree htree_dirblock_to_tree ext4_read_dirblock __ext4_read_dirblock ext4_dirblock_csum_verify warn_no_space_for_csum __warn_no_space_for_csum return ERR_PTR(-EFSBADCRC) // -EBADMSG -74 nfserrno // WARNING [ 161.115610] ------------[ cut here ]------------ [ 161.116465] nfsd: non-standard errno: -74 [ 161.117315] WARNING: CPU: 1 PID: 780 at fs/nfsd/nfsproc.c:878 nfserrno+0x9d/0xd0 [ 161.118596] Modules linked in: [ 161.119243] CPU: 1 PID: 780 Comm: nfsd Not tainted 5.10.0-00014-g79679361fd5d #138 [ 161.120684] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.14.0-0-g155821a1990b-prebuilt.qe mu.org 04/01/2014 [ 161.123601] RIP: 0010:nfserrno+0x9d/0xd0 [ 161.124676] Code: 0f 87 da 30 dd 00 83 e3 01 b8 00 00 00 05 75 d7 44 89 ee 48 c7 c7 c0 57 24 98 89 44 24 04 c6 05 ce 2b 61 03 01 e8 99 20 d8 00 <0f> 0b 8b 44 24 04 eb b5 4c 89 e6 48 c7 c7 a0 6d a4 99 e8 cc 15 33 [ 161.127797] RSP: 0018:ffffc90000e2f9c0 EFLAGS: 00010286 [ 161.128794] RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000000000 [ 161.130089] RDX: 1ffff1103ee16f6d RSI: 0000000000000008 RDI: fffff520001c5f2a [ 161.131379] RBP: 0000000000000022 R08: 0000000000000001 R09: ffff8881f70c1827 [ 161.132664] R10: ffffed103ee18304 R11: 0000000000000001 R12: 0000000000000021 [ 161.133949] R13: 00000000ffffffb6 R14: ffff8881317c0000 R15: ffffc90000e2fbd8 [ 161.135244] FS: 0000000000000000(0000) GS:ffff8881f7080000(0000) knlGS:0000000000000000 [ 161.136695] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 161.137761] CR2: 00007fcaad70b348 CR3: 0000000144256006 CR4: 0000000000770ee0 [ 161.139041] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 161.140291] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 161.141519] PKRU: 55555554 [ 161.142076] Call Trace: [ 161.142575] ? __warn+0x9b/0x140 [ 161.143229] ? nfserrno+0x9d/0xd0 [ 161.143872] ? report_bug+0x125/0x150 [ 161.144595] ? handle_bug+0x41/0x90 [ 161.145284] ? exc_invalid_op+0x14/0x70 [ 161.146009] ? asm_exc_invalid_op+0x12/0x20 [ 161.146816] ? nfserrno+0x9d/0xd0 [ 161.147487] nfsd_buffered_readdir+0x28b/0x2b0 [ 161.148333] ? nfsd4_encode_dirent_fattr+0x380/0x380 [ 161.149258] ? nfsd_buffered_filldir+0xf0/0xf0 [ 161.150093] ? wait_for_concurrent_writes+0x170/0x170 [ 161.151004] ? generic_file_llseek_size+0x48/0x160 [ 161.151895] nfsd_readdir+0x132/0x190 [ 161.152606] ? nfsd4_encode_dirent_fattr+0x380/0x380 [ 161.153516] ? nfsd_unlink+0x380/0x380 [ 161.154256] ? override_creds+0x45/0x60 [ 161.155006] nfsd4_encode_readdir+0x21a/0x3d0 [ 161.155850] ? nfsd4_encode_readlink+0x210/0x210 [ 161.156731] ? write_bytes_to_xdr_buf+0x97/0xe0 [ 161.157598] ? __write_bytes_to_xdr_buf+0xd0/0xd0 [ 161.158494] ? lock_downgrade+0x90/0x90 [ 161.159232] ? nfs4svc_decode_voidarg+0x10/0x10 [ 161.160092] nfsd4_encode_operation+0x15a/0x440 [ 161.160959] nfsd4_proc_compound+0x718/0xe90 [ 161.161818] nfsd_dispatch+0x18e/0x2c0 [ 161.162586] svc_process_common+0x786/0xc50 [ 161.163403] ? nfsd_svc+0x380/0x380 [ 161.164137] ? svc_printk+0x160/0x160 [ 161.164846] ? svc_xprt_do_enqueue.part.0+0x365/0x380 [ 161.165808] ? nfsd_svc+0x380/0x380 [ 161.166523] ? rcu_is_watching+0x23/0x40 [ 161.167309] svc_process+0x1a5/0x200 [ 161.168019] nfsd+0x1f5/0x380 [ 161.168663] ? nfsd_shutdown_threads+0x260/0x260 [ 161.169554] kthread+0x1c4/0x210 [ 161.170224] ? kthread_insert_work_sanity_check+0x80/0x80 [ 161.171246] ret_from_fork+0x1f/0x30 Signed-off-by: Li Lingfeng Reviewed-by: Jeff Layton Cc: stable@vger.kernel.org Signed-off-by: Chuck Lever --- fs/nfsd/vfs.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c index ec99c91df173..22325b590e17 100644 --- a/fs/nfsd/vfs.c +++ b/fs/nfsd/vfs.c @@ -100,6 +100,7 @@ nfserrno (int errno) { nfserr_io, -EUCLEAN }, { nfserr_perm, -ENOKEY }, { nfserr_no_grace, -ENOGRACE}, + { nfserr_io, -EBADMSG }, }; int i; From c2feb7ee39151005786f202bb1a373d96b653cd6 Mon Sep 17 00:00:00 2001 From: Hongbo Li Date: Wed, 21 Aug 2024 14:53:26 +0800 Subject: [PATCH 239/302] nfsd: use LIST_HEAD() to simplify code list_head can be initialized automatically with LIST_HEAD() instead of calling INIT_LIST_HEAD(). Signed-off-by: Hongbo Li Signed-off-by: Chuck Lever --- fs/nfsd/nfs4state.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index fa58de23a753..b743a021b51a 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -1663,9 +1663,7 @@ static void release_openowner(struct nfs4_openowner *oo) { struct nfs4_ol_stateid *stp; struct nfs4_client *clp = oo->oo_owner.so_client; - struct list_head reaplist; - - INIT_LIST_HEAD(&reaplist); + LIST_HEAD(reaplist); spin_lock(&clp->cl_lock); unhash_openowner_locked(oo); @@ -2369,9 +2367,8 @@ __destroy_client(struct nfs4_client *clp) int i; struct nfs4_openowner *oo; struct nfs4_delegation *dp; - struct list_head reaplist; + LIST_HEAD(reaplist); - INIT_LIST_HEAD(&reaplist); spin_lock(&state_lock); while (!list_empty(&clp->cl_delegations)) { dp = list_entry(clp->cl_delegations.next, struct nfs4_delegation, dl_perclnt); @@ -6615,9 +6612,8 @@ deleg_reaper(struct nfsd_net *nn) { struct list_head *pos, *next; struct nfs4_client *clp; - struct list_head cblist; + LIST_HEAD(cblist); - INIT_LIST_HEAD(&cblist); spin_lock(&nn->client_lock); list_for_each_safe(pos, next, &nn->client_lru) { clp = list_entry(pos, struct nfs4_client, cl_lru); From eb059a413c06af1c59b26cb85ddb7cf73d82b92e Mon Sep 17 00:00:00 2001 From: Li Lingfeng Date: Fri, 23 Aug 2024 15:00:49 +0800 Subject: [PATCH 240/302] nfsd: remove unused parameter of nfsd_file_mark_find_or_create Commit 427f5f83a319 ("NFSD: Ensure nf_inode is never dereferenced") passes inode directly to nfsd_file_mark_find_or_create instead of getting it from nf, so there is no need to pass nf. Signed-off-by: Li Lingfeng Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/filecache.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/nfsd/filecache.c b/fs/nfsd/filecache.c index 9e9d246f993c..24e8f1fbcebb 100644 --- a/fs/nfsd/filecache.c +++ b/fs/nfsd/filecache.c @@ -152,7 +152,7 @@ nfsd_file_mark_put(struct nfsd_file_mark *nfm) } static struct nfsd_file_mark * -nfsd_file_mark_find_or_create(struct nfsd_file *nf, struct inode *inode) +nfsd_file_mark_find_or_create(struct inode *inode) { int err; struct fsnotify_mark *mark; @@ -1078,7 +1078,7 @@ out: open_file: trace_nfsd_file_alloc(nf); - nf->nf_mark = nfsd_file_mark_find_or_create(nf, inode); + nf->nf_mark = nfsd_file_mark_find_or_create(inode); if (nf->nf_mark) { if (file) { get_file(file); From 76a3f3f164d470ef1f22a34dbdb13549934cf339 Mon Sep 17 00:00:00 2001 From: Li Lingfeng Date: Sat, 24 Aug 2024 09:43:36 +0800 Subject: [PATCH 241/302] nfsd: fix some spelling errors in comments Fix spelling errors in comments of nfsd4_release_lockowner and nfs4_set_delegation. Signed-off-by: Li Lingfeng Signed-off-by: Chuck Lever --- fs/nfsd/nfs4state.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index b743a021b51a..070fdf60334f 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -5853,7 +5853,7 @@ nfs4_set_delegation(struct nfsd4_open *open, struct nfs4_ol_stateid *stp, /* * Now that the deleg is set, check again to ensure that nothing - * raced in and changed the mode while we weren't lookng. + * raced in and changed the mode while we weren't looking. */ status = nfsd4_verify_setuid_write(open, fp->fi_deleg_file); if (status) @@ -8326,7 +8326,7 @@ out: * @cstate: NFSv4 COMPOUND state * @u: RELEASE_LOCKOWNER arguments * - * Check if theree are any locks still held and if not - free the lockowner + * Check if there are any locks still held and if not, free the lockowner * and any lock state that is owned. * * Return values: From e8581a9124470b751db8ec46044720d1597dbbfe Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Mon, 26 Aug 2024 08:50:11 -0400 Subject: [PATCH 242/302] nfsd: add more info to WARN_ON_ONCE on failed callbacks Currently, you get the warning and stack trace, but nothing is printed about the relevant error codes. Add that in. Signed-off-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/nfs4callback.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c index d756f443fc44..dee9477cc5b5 100644 --- a/fs/nfsd/nfs4callback.c +++ b/fs/nfsd/nfs4callback.c @@ -1333,7 +1333,8 @@ static void nfsd4_cb_done(struct rpc_task *task, void *calldata) return; if (cb->cb_status) { - WARN_ON_ONCE(task->tk_status); + WARN_ONCE(task->tk_status, "cb_status=%d tk_status=%d", + cb->cb_status, task->tk_status); task->tk_status = cb->cb_status; } From c1c9f3ea7405b5069925ab2fd6326e3b980b28b0 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Mon, 26 Aug 2024 08:50:12 -0400 Subject: [PATCH 243/302] nfsd: track the main opcode for callbacks Keep track of the "main" opcode for the callback, and display it in the tracepoint. This makes it simpler to discern what's happening when there is more than one callback in flight. The one special case is the CB_NULL RPC. That's not a CB_COMPOUND opcode, so designate the value 0 for that. Signed-off-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/nfs4layouts.c | 1 + fs/nfsd/nfs4proc.c | 3 ++- fs/nfsd/nfs4state.c | 4 ++++ fs/nfsd/state.h | 1 + fs/nfsd/trace.h | 23 +++++++++++++++++++---- 5 files changed, 27 insertions(+), 5 deletions(-) diff --git a/fs/nfsd/nfs4layouts.c b/fs/nfsd/nfs4layouts.c index 4f3072b5979a..fbfddd3c4c94 100644 --- a/fs/nfsd/nfs4layouts.c +++ b/fs/nfsd/nfs4layouts.c @@ -740,6 +740,7 @@ static const struct nfsd4_callback_ops nfsd4_cb_layout_ops = { .prepare = nfsd4_cb_layout_prepare, .done = nfsd4_cb_layout_done, .release = nfsd4_cb_layout_release, + .opcode = OP_CB_LAYOUTRECALL, }; static bool diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index fc68af757080..963a02e179a0 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -1622,7 +1622,8 @@ static int nfsd4_cb_offload_done(struct nfsd4_callback *cb, static const struct nfsd4_callback_ops nfsd4_cb_offload_ops = { .release = nfsd4_cb_offload_release, - .done = nfsd4_cb_offload_done + .done = nfsd4_cb_offload_done, + .opcode = OP_CB_OFFLOAD, }; static void nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync) diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 070fdf60334f..d349e3308899 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -400,6 +400,7 @@ static const struct nfsd4_callback_ops nfsd4_cb_notify_lock_ops = { .prepare = nfsd4_cb_notify_lock_prepare, .done = nfsd4_cb_notify_lock_done, .release = nfsd4_cb_notify_lock_release, + .opcode = OP_CB_NOTIFY_LOCK, }; /* @@ -3083,11 +3084,13 @@ nfsd4_cb_getattr_release(struct nfsd4_callback *cb) static const struct nfsd4_callback_ops nfsd4_cb_recall_any_ops = { .done = nfsd4_cb_recall_any_done, .release = nfsd4_cb_recall_any_release, + .opcode = OP_CB_RECALL_ANY, }; static const struct nfsd4_callback_ops nfsd4_cb_getattr_ops = { .done = nfsd4_cb_getattr_done, .release = nfsd4_cb_getattr_release, + .opcode = OP_CB_GETATTR, }; static void nfs4_cb_getattr(struct nfs4_cb_fattr *ncf) @@ -5215,6 +5218,7 @@ static const struct nfsd4_callback_ops nfsd4_cb_recall_ops = { .prepare = nfsd4_cb_recall_prepare, .done = nfsd4_cb_recall_done, .release = nfsd4_cb_recall_release, + .opcode = OP_CB_RECALL, }; static void nfsd_break_one_deleg(struct nfs4_delegation *dp) diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h index ec4559ecd193..79c743c01a47 100644 --- a/fs/nfsd/state.h +++ b/fs/nfsd/state.h @@ -79,6 +79,7 @@ struct nfsd4_callback_ops { void (*prepare)(struct nfsd4_callback *); int (*done)(struct nfsd4_callback *, struct rpc_task *); void (*release)(struct nfsd4_callback *); + uint32_t opcode; }; /* diff --git a/fs/nfsd/trace.h b/fs/nfsd/trace.h index 77bbd23aa150..411927eaca1e 100644 --- a/fs/nfsd/trace.h +++ b/fs/nfsd/trace.h @@ -1553,6 +1553,19 @@ TRACE_EVENT(nfsd_cb_setup_err, __entry->error) ); +/* Not a real opcode, but there is no 0 operation. */ +#define _CB_NULL 0 + +#define show_nfsd_cb_opcode(val) \ + __print_symbolic(val, \ + { _CB_NULL, "CB_NULL" }, \ + { OP_CB_GETATTR, "CB_GETATTR" }, \ + { OP_CB_RECALL, "CB_RECALL" }, \ + { OP_CB_LAYOUTRECALL, "CB_LAYOUTRECALL" }, \ + { OP_CB_RECALL_ANY, "CB_RECALL_ANY" }, \ + { OP_CB_NOTIFY_LOCK, "CB_NOTIFY_LOCK" }, \ + { OP_CB_OFFLOAD, "CB_OFFLOAD" }) + DECLARE_EVENT_CLASS(nfsd_cb_lifetime_class, TP_PROTO( const struct nfs4_client *clp, @@ -1563,6 +1576,7 @@ DECLARE_EVENT_CLASS(nfsd_cb_lifetime_class, __field(u32, cl_boot) __field(u32, cl_id) __field(const void *, cb) + __field(unsigned long, opcode) __field(bool, need_restart) __sockaddr(addr, clp->cl_cb_conn.cb_addrlen) ), @@ -1570,14 +1584,15 @@ DECLARE_EVENT_CLASS(nfsd_cb_lifetime_class, __entry->cl_boot = clp->cl_clientid.cl_boot; __entry->cl_id = clp->cl_clientid.cl_id; __entry->cb = cb; + __entry->opcode = cb->cb_ops ? cb->cb_ops->opcode : _CB_NULL; __entry->need_restart = cb->cb_need_restart; __assign_sockaddr(addr, &clp->cl_cb_conn.cb_addr, clp->cl_cb_conn.cb_addrlen) ), - TP_printk("addr=%pISpc client %08x:%08x cb=%p%s", - __get_sockaddr(addr), __entry->cl_boot, __entry->cl_id, - __entry->cb, __entry->need_restart ? - " (need restart)" : " (first try)" + TP_printk("addr=%pISpc client %08x:%08x cb=%p%s opcode=%s", + __get_sockaddr(addr), __entry->cl_boot, __entry->cl_id, __entry->cb, + __entry->need_restart ? " (need restart)" : " (first try)", + show_nfsd_cb_opcode(__entry->opcode) ) ); From ba017fd391e05aba3d8a96c06d0287ffeeb8458c Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Mon, 26 Aug 2024 08:50:13 -0400 Subject: [PATCH 244/302] nfsd: add more nfsd_cb tracepoints Add some tracepoints in the callback client RPC operations. Also add a tracepoint to nfsd4_cb_getattr_done. Signed-off-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/nfs4callback.c | 5 +++++ fs/nfsd/nfs4state.c | 3 +++ fs/nfsd/trace.h | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c index dee9477cc5b5..b5b3ab9d719a 100644 --- a/fs/nfsd/nfs4callback.c +++ b/fs/nfsd/nfs4callback.c @@ -1223,6 +1223,7 @@ static void nfsd4_cb_prepare(struct rpc_task *task, void *calldata) * cb_seq_status is only set in decode_cb_sequence4res, * and so will remain 1 if an rpc level failure occurs. */ + trace_nfsd_cb_rpc_prepare(clp); cb->cb_seq_status = 1; cb->cb_status = 0; if (minorversion && !nfsd41_cb_get_slot(cb, task)) @@ -1329,6 +1330,8 @@ static void nfsd4_cb_done(struct rpc_task *task, void *calldata) struct nfsd4_callback *cb = calldata; struct nfs4_client *clp = cb->cb_clp; + trace_nfsd_cb_rpc_done(clp); + if (!nfsd4_cb_sequence_done(task, cb)) return; @@ -1360,6 +1363,8 @@ static void nfsd4_cb_release(void *calldata) { struct nfsd4_callback *cb = calldata; + trace_nfsd_cb_rpc_release(cb->cb_clp); + if (cb->cb_need_restart) nfsd4_queue_cb(cb); else diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index d349e3308899..eaa11d42d1b1 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -3057,7 +3057,10 @@ nfsd4_cb_getattr_done(struct nfsd4_callback *cb, struct rpc_task *task) { struct nfs4_cb_fattr *ncf = container_of(cb, struct nfs4_cb_fattr, ncf_getattr); + struct nfs4_delegation *dp = + container_of(ncf, struct nfs4_delegation, dl_cb_fattr); + trace_nfsd_cb_getattr_done(&dp->dl_stid.sc_stateid, task); ncf->ncf_cb_status = task->tk_status; switch (task->tk_status) { case -NFS4ERR_DELAY: diff --git a/fs/nfsd/trace.h b/fs/nfsd/trace.h index 411927eaca1e..5c2d933dc469 100644 --- a/fs/nfsd/trace.h +++ b/fs/nfsd/trace.h @@ -1486,6 +1486,9 @@ DEFINE_NFSD_CB_EVENT(new_state); DEFINE_NFSD_CB_EVENT(probe); DEFINE_NFSD_CB_EVENT(lost); DEFINE_NFSD_CB_EVENT(shutdown); +DEFINE_NFSD_CB_EVENT(rpc_prepare); +DEFINE_NFSD_CB_EVENT(rpc_done); +DEFINE_NFSD_CB_EVENT(rpc_release); TRACE_DEFINE_ENUM(RPC_AUTH_NULL); TRACE_DEFINE_ENUM(RPC_AUTH_UNIX); @@ -1845,6 +1848,7 @@ DEFINE_NFSD_CB_DONE_EVENT(nfsd_cb_recall_done); DEFINE_NFSD_CB_DONE_EVENT(nfsd_cb_notify_lock_done); DEFINE_NFSD_CB_DONE_EVENT(nfsd_cb_layout_done); DEFINE_NFSD_CB_DONE_EVENT(nfsd_cb_offload_done); +DEFINE_NFSD_CB_DONE_EVENT(nfsd_cb_getattr_done); TRACE_EVENT(nfsd_cb_recall_any_done, TP_PROTO( From d078cbf5c38de83bc31f83c47dcd2184c04a50c7 Mon Sep 17 00:00:00 2001 From: Guoqing Jiang Date: Wed, 21 Aug 2024 22:03:18 +0800 Subject: [PATCH 245/302] nfsd: call cache_put if xdr_reserve_space returns NULL If not enough buffer space available, but idmap_lookup has triggered lookup_fn which calls cache_get and returns successfully. Then we missed to call cache_put here which pairs with cache_get. Fixes: ddd1ea563672 ("nfsd4: use xdr_reserve_space in attribute encoding") Signed-off-by: Guoqing Jiang Reviwed-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/nfs4idmap.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/fs/nfsd/nfs4idmap.c b/fs/nfsd/nfs4idmap.c index 7a806ac13e31..8cca1329f348 100644 --- a/fs/nfsd/nfs4idmap.c +++ b/fs/nfsd/nfs4idmap.c @@ -581,6 +581,7 @@ static __be32 idmap_id_to_name(struct xdr_stream *xdr, .id = id, .type = type, }; + __be32 status = nfs_ok; __be32 *p; int ret; struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); @@ -593,12 +594,16 @@ static __be32 idmap_id_to_name(struct xdr_stream *xdr, return nfserrno(ret); ret = strlen(item->name); WARN_ON_ONCE(ret > IDMAP_NAMESZ); + p = xdr_reserve_space(xdr, ret + 4); - if (!p) - return nfserr_resource; - p = xdr_encode_opaque(p, item->name, ret); + if (unlikely(!p)) { + status = nfserr_resource; + goto out_put; + } + xdr_encode_opaque(p, item->name, ret); +out_put: cache_put(&item->h, nn->idtoname_cache); - return 0; + return status; } static bool From 2869b3a00e202ee8d3d90ab7d88f4057c4bb3135 Mon Sep 17 00:00:00 2001 From: Thorsten Blum Date: Wed, 28 Aug 2024 23:42:55 +0200 Subject: [PATCH 246/302] NFSD: Annotate struct pnfs_block_deviceaddr with __counted_by() Add the __counted_by compiler attribute to the flexible array member volumes to improve access bounds-checking via CONFIG_UBSAN_BOUNDS and CONFIG_FORTIFY_SOURCE. Use struct_size() instead of manually calculating the number of bytes to allocate for a pnfs_block_deviceaddr with a single volume. Signed-off-by: Thorsten Blum Reviewed-by: Gustavo A. R. Silva Acked-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/blocklayout.c | 6 ++---- fs/nfsd/blocklayoutxdr.h | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/fs/nfsd/blocklayout.c b/fs/nfsd/blocklayout.c index 3c040c81c77d..08a20e5bcf7f 100644 --- a/fs/nfsd/blocklayout.c +++ b/fs/nfsd/blocklayout.c @@ -147,8 +147,7 @@ nfsd4_block_get_device_info_simple(struct super_block *sb, struct pnfs_block_deviceaddr *dev; struct pnfs_block_volume *b; - dev = kzalloc(sizeof(struct pnfs_block_deviceaddr) + - sizeof(struct pnfs_block_volume), GFP_KERNEL); + dev = kzalloc(struct_size(dev, volumes, 1), GFP_KERNEL); if (!dev) return -ENOMEM; gdp->gd_device = dev; @@ -255,8 +254,7 @@ nfsd4_block_get_device_info_scsi(struct super_block *sb, const struct pr_ops *ops; int ret; - dev = kzalloc(sizeof(struct pnfs_block_deviceaddr) + - sizeof(struct pnfs_block_volume), GFP_KERNEL); + dev = kzalloc(struct_size(dev, volumes, 1), GFP_KERNEL); if (!dev) return -ENOMEM; gdp->gd_device = dev; diff --git a/fs/nfsd/blocklayoutxdr.h b/fs/nfsd/blocklayoutxdr.h index b0361e8aa9a7..4e28ac8f1127 100644 --- a/fs/nfsd/blocklayoutxdr.h +++ b/fs/nfsd/blocklayoutxdr.h @@ -47,7 +47,7 @@ struct pnfs_block_volume { struct pnfs_block_deviceaddr { u32 nr_volumes; - struct pnfs_block_volume volumes[]; + struct pnfs_block_volume volumes[] __counted_by(nr_volumes); }; __be32 nfsd4_block_encode_getdeviceinfo(struct xdr_stream *xdr, From aeddf8e6c5662d60d434ce59f7e08ea020162323 Mon Sep 17 00:00:00 2001 From: Yan Zhen Date: Fri, 30 Aug 2024 09:43:56 +0800 Subject: [PATCH 247/302] sunrpc: xprtrdma: Use ERR_CAST() to return Using ERR_CAST() is more reasonable and safer, When it is necessary to convert the type of an error pointer and return it. Signed-off-by: Yan Zhen Signed-off-by: Chuck Lever --- net/sunrpc/xprtrdma/svc_rdma_transport.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c index 581cc5ed7c0c..c3fbf0779d4a 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_transport.c +++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c @@ -369,7 +369,7 @@ static struct svc_xprt *svc_rdma_create(struct svc_serv *serv, listen_id = svc_rdma_create_listen_id(net, sa, cma_xprt); if (IS_ERR(listen_id)) { kfree(cma_xprt); - return (struct svc_xprt *)listen_id; + return ERR_CAST(listen_id); } cma_xprt->sc_cm_id = listen_id; From 985eeae9c874157bb8ece9060020856f68aa08d5 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Fri, 30 Aug 2024 17:03:16 +1000 Subject: [PATCH 248/302] nfsd: use clear_and_wake_up_bit() nfsd has two places that open-code clear_and_wake_up_bit(). One has the required memory barriers. The other does not. Change both to use clear_and_wake_up_bit() so we have the barriers without the noise. Signed-off-by: NeilBrown Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/nfs4recover.c | 5 +---- fs/nfsd/nfs4state.c | 3 +-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/fs/nfsd/nfs4recover.c b/fs/nfsd/nfs4recover.c index 67d8673a9391..0d25ad5eeae3 100644 --- a/fs/nfsd/nfs4recover.c +++ b/fs/nfsd/nfs4recover.c @@ -1895,10 +1895,7 @@ nfsd4_cltrack_upcall_lock(struct nfs4_client *clp) static void nfsd4_cltrack_upcall_unlock(struct nfs4_client *clp) { - smp_mb__before_atomic(); - clear_bit(NFSD4_CLIENT_UPCALL_LOCK, &clp->cl_flags); - smp_mb__after_atomic(); - wake_up_bit(&clp->cl_flags, NFSD4_CLIENT_UPCALL_LOCK); + clear_and_wake_up_bit(NFSD4_CLIENT_UPCALL_LOCK, &clp->cl_flags); } static void diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index eaa11d42d1b1..0ea70d615c78 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -3079,8 +3079,7 @@ nfsd4_cb_getattr_release(struct nfsd4_callback *cb) struct nfs4_delegation *dp = container_of(ncf, struct nfs4_delegation, dl_cb_fattr); - clear_bit(CB_GETATTR_BUSY, &ncf->ncf_cb_flags); - wake_up_bit(&ncf->ncf_cb_flags, CB_GETATTR_BUSY); + clear_and_wake_up_bit(CB_GETATTR_BUSY, &ncf->ncf_cb_flags); nfs4_put_stid(&dp->dl_stid); } From 15392c8cd13449bf0c09faf8e78ca2fdfb023284 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Fri, 30 Aug 2024 17:03:17 +1000 Subject: [PATCH 249/302] nfsd: avoid races with wake_up_var() wake_up_var() needs a barrier after the important change is made in the var and before wake_up_var() is called, else it is possible that a wake up won't be sent when it should. In each case here the var is changed in an "atomic" manner, so smb_mb__after_atomic() is sufficient. In one case the important change (removing the lease) is performed *after* the wake_up, which is backwards. The code survives in part because the wait_var_event is given a timeout. This patch adds the required barriers and calls destroy_delegation() *before* waking any threads waiting for the delegation to be destroyed. Signed-off-by: NeilBrown Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/nfs4state.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 0ea70d615c78..7ade551bc022 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -4706,6 +4706,7 @@ void nfsd4_cstate_clear_replay(struct nfsd4_compound_state *cstate) if (so != NULL) { cstate->replay_owner = NULL; atomic_set(&so->so_replay.rp_locked, RP_UNLOCKED); + smp_mb__after_atomic(); wake_up_var(&so->so_replay.rp_locked); nfs4_put_stateowner(so); } @@ -5006,6 +5007,7 @@ move_to_close_lru(struct nfs4_ol_stateid *s, struct net *net) * so tell them to stop waiting. */ atomic_set(&oo->oo_owner.so_replay.rp_locked, RP_UNHASHED); + smp_mb__after_atomic(); wake_up_var(&oo->oo_owner.so_replay.rp_locked); wait_event(close_wq, refcount_read(&s->st_stid.sc_count) == 2); @@ -7475,8 +7477,9 @@ nfsd4_delegreturn(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, goto put_stateid; trace_nfsd_deleg_return(stateid); - wake_up_var(d_inode(cstate->current_fh.fh_dentry)); destroy_delegation(dp); + smp_mb__after_atomic(); + wake_up_var(d_inode(cstate->current_fh.fh_dentry)); put_stateid: nfs4_put_stid(&dp->dl_stid); out: From 9ed666eba4e0a2bb8ffaa3739d830b64d4f2aaad Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Wed, 28 Aug 2024 13:40:03 -0400 Subject: [PATCH 250/302] NFSD: Async COPY result needs to return a write verifier Currently, when NFSD handles an asynchronous COPY, it returns a zero write verifier, relying on the subsequent CB_OFFLOAD callback to pass the write verifier and a stable_how4 value to the client. However, if the CB_OFFLOAD never arrives at the client (for example, if a network partition occurs just as the server sends the CB_OFFLOAD operation), the client will never receive this verifier. Thus, if the client sends a follow-up COMMIT, there is no way for the client to assess the COMMIT result. The usual recovery for a missing CB_OFFLOAD is for the client to send an OFFLOAD_STATUS operation, but that operation does not carry a write verifier in its result. Neither does it carry a stable_how4 value, so the client /must/ send a COMMIT in this case -- which will always fail because currently there's still no write verifier in the COPY result. Thus the server needs to return a normal write verifier in its COPY result even if the COPY operation is to be performed asynchronously. If the server recognizes the callback stateid in subsequent OFFLOAD_STATUS operations, then obviously it has not restarted, and the write verifier the client received in the COPY result is still valid and can be used to assess a COMMIT of the copied data, if one is needed. Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/nfs4proc.c | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index 963a02e179a0..231c6035602f 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -752,15 +752,6 @@ nfsd4_access(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, &access->ac_supported); } -static void gen_boot_verifier(nfs4_verifier *verifier, struct net *net) -{ - __be32 *verf = (__be32 *)verifier->data; - - BUILD_BUG_ON(2*sizeof(*verf) != sizeof(verifier->data)); - - nfsd_copy_write_verifier(verf, net_generic(net, nfsd_net_id)); -} - static __be32 nfsd4_commit(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, union nfsd4_op_u *u) @@ -1632,7 +1623,6 @@ static void nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync) test_bit(NFSD4_COPY_F_COMMITTED, ©->cp_flags) ? NFS_FILE_SYNC : NFS_UNSTABLE; nfsd4_copy_set_sync(copy, sync); - gen_boot_verifier(©->cp_res.wr_verifier, copy->cp_clp->net); } static ssize_t _nfsd_copy_file_range(struct nfsd4_copy *copy, @@ -1805,9 +1795,11 @@ static __be32 nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, union nfsd4_op_u *u) { - struct nfsd4_copy *copy = &u->copy; - __be32 status; + struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); struct nfsd4_copy *async_copy = NULL; + struct nfsd4_copy *copy = &u->copy; + struct nfsd42_write_res *result; + __be32 status; /* * Currently, async COPY is not reliable. Force all COPY @@ -1816,6 +1808,9 @@ nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, */ nfsd4_copy_set_sync(copy, true); + result = ©->cp_res; + nfsd_copy_write_verifier((__be32 *)&result->wr_verifier.data, nn); + copy->cp_clp = cstate->clp; if (nfsd4_ssc_is_inter(copy)) { trace_nfsd_copy_inter(copy); @@ -1840,8 +1835,6 @@ nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, memcpy(©->fh, &cstate->current_fh.fh_handle, sizeof(struct knfsd_fh)); if (nfsd4_copy_is_async(copy)) { - struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); - status = nfserrno(-ENOMEM); async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL); if (!async_copy) @@ -1853,8 +1846,8 @@ nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, goto out_err; if (!nfs4_init_copy_state(nn, copy)) goto out_err; - memcpy(©->cp_res.cb_stateid, ©->cp_stateid.cs_stid, - sizeof(copy->cp_res.cb_stateid)); + memcpy(&result->cb_stateid, ©->cp_stateid.cs_stid, + sizeof(result->cb_stateid)); dup_copy_fields(copy, async_copy); async_copy->copy_task = kthread_create(nfsd4_do_async_copy, async_copy, "%s", "copy thread"); From aadc3bbea163b6caaaebfdd2b6c4667fbc726752 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Wed, 28 Aug 2024 13:40:04 -0400 Subject: [PATCH 251/302] NFSD: Limit the number of concurrent async COPY operations Nothing appears to limit the number of concurrent async COPY operations that clients can start. In addition, AFAICT each async COPY can copy an unlimited number of 4MB chunks, so can run for a long time. Thus IMO async COPY can become a DoS vector. Add a restriction mechanism that bounds the number of concurrent background COPY operations. Start simple and try to be fair -- this patch implements a per-namespace limit. An async COPY request that occurs while this limit is exceeded gets NFS4ERR_DELAY. The requesting client can choose to send the request again after a delay or fall back to a traditional read/write style copy. If there is need to make the mechanism more sophisticated, we can visit that in future patches. Cc: stable@vger.kernel.org Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/netns.h | 1 + fs/nfsd/nfs4proc.c | 11 +++++++++-- fs/nfsd/nfs4state.c | 1 + fs/nfsd/xdr4.h | 1 + 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h index 238fc4e56e53..37b8bfdcfeea 100644 --- a/fs/nfsd/netns.h +++ b/fs/nfsd/netns.h @@ -148,6 +148,7 @@ struct nfsd_net { u32 s2s_cp_cl_id; struct idr s2s_cp_stateids; spinlock_t s2s_cp_lock; + atomic_t pending_async_copies; /* * Version information diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index 231c6035602f..9655acb407b7 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -1280,6 +1280,7 @@ static void nfs4_put_copy(struct nfsd4_copy *copy) { if (!refcount_dec_and_test(©->refcount)) return; + atomic_dec(©->cp_nn->pending_async_copies); kfree(copy->cp_src); kfree(copy); } @@ -1835,10 +1836,16 @@ nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, memcpy(©->fh, &cstate->current_fh.fh_handle, sizeof(struct knfsd_fh)); if (nfsd4_copy_is_async(copy)) { - status = nfserrno(-ENOMEM); async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL); if (!async_copy) goto out_err; + async_copy->cp_nn = nn; + /* Arbitrary cap on number of pending async copy operations */ + if (atomic_inc_return(&nn->pending_async_copies) > + (int)rqstp->rq_pool->sp_nrthreads) { + atomic_dec(&nn->pending_async_copies); + goto out_err; + } INIT_LIST_HEAD(&async_copy->copies); refcount_set(&async_copy->refcount, 1); async_copy->cp_src = kmalloc(sizeof(*async_copy->cp_src), GFP_KERNEL); @@ -1878,7 +1885,7 @@ out_err: } if (async_copy) cleanup_async_copy(async_copy); - status = nfserrno(-ENOMEM); + status = nfserr_jukebox; goto out; } diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 7ade551bc022..a49aa75bc0b7 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -8554,6 +8554,7 @@ static int nfs4_state_create_net(struct net *net) spin_lock_init(&nn->client_lock); spin_lock_init(&nn->s2s_cp_lock); idr_init(&nn->s2s_cp_stateids); + atomic_set(&nn->pending_async_copies, 0); spin_lock_init(&nn->blocked_locks_lock); INIT_LIST_HEAD(&nn->blocked_locks_lru); diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h index fbdd42cde1fa..2a21a7662e03 100644 --- a/fs/nfsd/xdr4.h +++ b/fs/nfsd/xdr4.h @@ -713,6 +713,7 @@ struct nfsd4_copy { struct nfsd4_ssc_umount_item *ss_nsui; struct nfs_fh c_fh; nfs4_stateid stateid; + struct nfsd_net *cp_nn; }; static inline void nfsd4_copy_set_sync(struct nfsd4_copy *copy, bool sync) From 11848e985ca03deec5ee3552c8f8d34541dd8d35 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Wed, 28 Aug 2024 13:40:05 -0400 Subject: [PATCH 252/302] NFSD: Display copy stateids with conventional print formatting Make it easier to grep for s2s COPY stateids in trace logs: Use the same display format in nfsd_copy_class as is used to display other stateids. Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/trace.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/fs/nfsd/trace.h b/fs/nfsd/trace.h index 5c2d933dc469..09770dcbd448 100644 --- a/fs/nfsd/trace.h +++ b/fs/nfsd/trace.h @@ -2172,14 +2172,14 @@ DECLARE_EVENT_CLASS(nfsd_copy_class, sizeof(struct sockaddr_in6)); ), TP_printk("client=%pISpc intra=%d async=%d " - "src_stateid[si_generation:0x%x cl_boot:0x%x cl_id:0x%x so_id:0x%x] " - "dst_stateid[si_generation:0x%x cl_boot:0x%x cl_id:0x%x so_id:0x%x] " + "src_client %08x:%08x src_stateid %08x:%08x " + "dst_client %08x:%08x dst_stateid %08x:%08x " "cp_src_pos=%llu cp_dst_pos=%llu cp_count=%llu", __get_sockaddr(addr), __entry->intra, __entry->async, - __entry->src_si_generation, __entry->src_cl_boot, - __entry->src_cl_id, __entry->src_so_id, - __entry->dst_si_generation, __entry->dst_cl_boot, - __entry->dst_cl_id, __entry->dst_so_id, + __entry->src_cl_boot, __entry->src_cl_id, + __entry->src_so_id, __entry->src_si_generation, + __entry->dst_cl_boot, __entry->dst_cl_id, + __entry->dst_so_id, __entry->dst_si_generation, __entry->src_cp_pos, __entry->dst_cp_pos, __entry->cp_count ) ); From e1d2697c53c0a74ece4402b2ae0047bfdaddb89b Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Wed, 28 Aug 2024 13:40:06 -0400 Subject: [PATCH 253/302] NFSD: Record the callback stateid in copy tracepoints Match COPY operations up with CB_OFFLOAD operations. Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/trace.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/fs/nfsd/trace.h b/fs/nfsd/trace.h index 09770dcbd448..922c3452749a 100644 --- a/fs/nfsd/trace.h +++ b/fs/nfsd/trace.h @@ -2146,6 +2146,10 @@ DECLARE_EVENT_CLASS(nfsd_copy_class, __field(u32, dst_cl_id) __field(u32, dst_so_id) __field(u32, dst_si_generation) + __field(u32, cb_cl_boot) + __field(u32, cb_cl_id) + __field(u32, cb_so_id) + __field(u32, cb_si_generation) __field(u64, src_cp_pos) __field(u64, dst_cp_pos) __field(u64, cp_count) @@ -2154,6 +2158,7 @@ DECLARE_EVENT_CLASS(nfsd_copy_class, TP_fast_assign( const stateid_t *src_stp = ©->cp_src_stateid; const stateid_t *dst_stp = ©->cp_dst_stateid; + const stateid_t *cb_stp = ©->cp_res.cb_stateid; __entry->intra = test_bit(NFSD4_COPY_F_INTRA, ©->cp_flags); __entry->async = !test_bit(NFSD4_COPY_F_SYNCHRONOUS, ©->cp_flags); @@ -2165,6 +2170,10 @@ DECLARE_EVENT_CLASS(nfsd_copy_class, __entry->dst_cl_id = dst_stp->si_opaque.so_clid.cl_id; __entry->dst_so_id = dst_stp->si_opaque.so_id; __entry->dst_si_generation = dst_stp->si_generation; + __entry->cb_cl_boot = cb_stp->si_opaque.so_clid.cl_boot; + __entry->cb_cl_id = cb_stp->si_opaque.so_clid.cl_id; + __entry->cb_so_id = cb_stp->si_opaque.so_id; + __entry->cb_si_generation = cb_stp->si_generation; __entry->src_cp_pos = copy->cp_src_pos; __entry->dst_cp_pos = copy->cp_dst_pos; __entry->cp_count = copy->cp_count; @@ -2174,12 +2183,15 @@ DECLARE_EVENT_CLASS(nfsd_copy_class, TP_printk("client=%pISpc intra=%d async=%d " "src_client %08x:%08x src_stateid %08x:%08x " "dst_client %08x:%08x dst_stateid %08x:%08x " + "cb_client %08x:%08x cb_stateid %08x:%08x " "cp_src_pos=%llu cp_dst_pos=%llu cp_count=%llu", __get_sockaddr(addr), __entry->intra, __entry->async, __entry->src_cl_boot, __entry->src_cl_id, __entry->src_so_id, __entry->src_si_generation, __entry->dst_cl_boot, __entry->dst_cl_id, __entry->dst_so_id, __entry->dst_si_generation, + __entry->cb_cl_boot, __entry->cb_cl_id, + __entry->cb_so_id, __entry->cb_si_generation, __entry->src_cp_pos, __entry->dst_cp_pos, __entry->cp_count ) ); From d3c430aa97e547614808831d28f43db26a854457 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Wed, 28 Aug 2024 13:40:07 -0400 Subject: [PATCH 254/302] NFSD: Clean up extra whitespace in trace_nfsd_copy_done Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/trace.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/nfsd/trace.h b/fs/nfsd/trace.h index 922c3452749a..fe6716d9fad4 100644 --- a/fs/nfsd/trace.h +++ b/fs/nfsd/trace.h @@ -2224,7 +2224,7 @@ TRACE_EVENT(nfsd_copy_done, __assign_sockaddr(addr, ©->cp_clp->cl_addr, sizeof(struct sockaddr_in6)); ), - TP_printk("addr=%pISpc status=%d intra=%d async=%d ", + TP_printk("addr=%pISpc status=%d intra=%d async=%d", __get_sockaddr(addr), __entry->status, __entry->intra, __entry->async ) ); From 0505de9615fa3d216828bfa9ddf502e918675986 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Wed, 28 Aug 2024 13:40:09 -0400 Subject: [PATCH 255/302] NFSD: Wrap async copy operations with trace points Add an nfsd_copy_async_done to record the timestamp, the final status code, and the callback stateid of an async copy. Rename the nfsd_copy_do_async tracepoint to match that naming convention to make it easier to enable both of these with a single glob. Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/nfs4proc.c | 3 +- fs/nfsd/trace.h | 71 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index 9655acb407b7..b5a6bf4f459f 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -1760,7 +1760,7 @@ static int nfsd4_do_async_copy(void *data) { struct nfsd4_copy *copy = (struct nfsd4_copy *)data; - trace_nfsd_copy_do_async(copy); + trace_nfsd_copy_async(copy); if (nfsd4_ssc_is_inter(copy)) { struct file *filp; @@ -1787,6 +1787,7 @@ static int nfsd4_do_async_copy(void *data) do_callback: set_bit(NFSD4_COPY_F_COMPLETED, ©->cp_flags); + trace_nfsd_copy_async_done(copy); nfsd4_send_cb_offload(copy); cleanup_async_copy(copy); return 0; diff --git a/fs/nfsd/trace.h b/fs/nfsd/trace.h index fe6716d9fad4..7ab66497e261 100644 --- a/fs/nfsd/trace.h +++ b/fs/nfsd/trace.h @@ -2203,7 +2203,7 @@ DEFINE_EVENT(nfsd_copy_class, nfsd_copy_##name, \ DEFINE_COPY_EVENT(inter); DEFINE_COPY_EVENT(intra); -DEFINE_COPY_EVENT(do_async); +DEFINE_COPY_EVENT(async); TRACE_EVENT(nfsd_copy_done, TP_PROTO( @@ -2229,6 +2229,75 @@ TRACE_EVENT(nfsd_copy_done, ) ); +TRACE_EVENT(nfsd_copy_async_done, + TP_PROTO( + const struct nfsd4_copy *copy + ), + TP_ARGS(copy), + TP_STRUCT__entry( + __field(int, status) + __field(bool, intra) + __field(bool, async) + __field(u32, src_cl_boot) + __field(u32, src_cl_id) + __field(u32, src_so_id) + __field(u32, src_si_generation) + __field(u32, dst_cl_boot) + __field(u32, dst_cl_id) + __field(u32, dst_so_id) + __field(u32, dst_si_generation) + __field(u32, cb_cl_boot) + __field(u32, cb_cl_id) + __field(u32, cb_so_id) + __field(u32, cb_si_generation) + __field(u64, src_cp_pos) + __field(u64, dst_cp_pos) + __field(u64, cp_count) + __sockaddr(addr, sizeof(struct sockaddr_in6)) + ), + TP_fast_assign( + const stateid_t *src_stp = ©->cp_src_stateid; + const stateid_t *dst_stp = ©->cp_dst_stateid; + const stateid_t *cb_stp = ©->cp_res.cb_stateid; + + __entry->status = be32_to_cpu(copy->nfserr); + __entry->intra = test_bit(NFSD4_COPY_F_INTRA, ©->cp_flags); + __entry->async = !test_bit(NFSD4_COPY_F_SYNCHRONOUS, ©->cp_flags); + __entry->src_cl_boot = src_stp->si_opaque.so_clid.cl_boot; + __entry->src_cl_id = src_stp->si_opaque.so_clid.cl_id; + __entry->src_so_id = src_stp->si_opaque.so_id; + __entry->src_si_generation = src_stp->si_generation; + __entry->dst_cl_boot = dst_stp->si_opaque.so_clid.cl_boot; + __entry->dst_cl_id = dst_stp->si_opaque.so_clid.cl_id; + __entry->dst_so_id = dst_stp->si_opaque.so_id; + __entry->dst_si_generation = dst_stp->si_generation; + __entry->cb_cl_boot = cb_stp->si_opaque.so_clid.cl_boot; + __entry->cb_cl_id = cb_stp->si_opaque.so_clid.cl_id; + __entry->cb_so_id = cb_stp->si_opaque.so_id; + __entry->cb_si_generation = cb_stp->si_generation; + __entry->src_cp_pos = copy->cp_src_pos; + __entry->dst_cp_pos = copy->cp_dst_pos; + __entry->cp_count = copy->cp_count; + __assign_sockaddr(addr, ©->cp_clp->cl_addr, + sizeof(struct sockaddr_in6)); + ), + TP_printk("client=%pISpc status=%d intra=%d async=%d " + "src_client %08x:%08x src_stateid %08x:%08x " + "dst_client %08x:%08x dst_stateid %08x:%08x " + "cb_client %08x:%08x cb_stateid %08x:%08x " + "cp_src_pos=%llu cp_dst_pos=%llu cp_count=%llu", + __get_sockaddr(addr), + __entry->status, __entry->intra, __entry->async, + __entry->src_cl_boot, __entry->src_cl_id, + __entry->src_so_id, __entry->src_si_generation, + __entry->dst_cl_boot, __entry->dst_cl_id, + __entry->dst_so_id, __entry->dst_si_generation, + __entry->cb_cl_boot, __entry->cb_cl_id, + __entry->cb_so_id, __entry->cb_si_generation, + __entry->src_cp_pos, __entry->dst_cp_pos, __entry->cp_count + ) +); + #endif /* _NFSD_TRACE_H */ #undef TRACE_INCLUDE_PATH From 22451a16b7ab7debefce660672566be887db1637 Mon Sep 17 00:00:00 2001 From: Li Lingfeng Date: Tue, 3 Sep 2024 19:14:46 +0800 Subject: [PATCH 256/302] nfsd: return -EINVAL when namelen is 0 When we have a corrupted main.sqlite in /var/lib/nfs/nfsdcld/, it may result in namelen being 0, which will cause memdup_user() to return ZERO_SIZE_PTR. When we access the name.data that has been assigned the value of ZERO_SIZE_PTR in nfs4_client_to_reclaim(), null pointer dereference is triggered. [ T1205] ================================================================== [ T1205] BUG: KASAN: null-ptr-deref in nfs4_client_to_reclaim+0xe9/0x260 [ T1205] Read of size 1 at addr 0000000000000010 by task nfsdcld/1205 [ T1205] [ T1205] CPU: 11 PID: 1205 Comm: nfsdcld Not tainted 5.10.0-00003-g2c1423731b8d #406 [ T1205] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS ?-20190727_073836-buildvm-ppc64le-16.ppc.fedoraproject.org-3.fc31 04/01/2014 [ T1205] Call Trace: [ T1205] dump_stack+0x9a/0xd0 [ T1205] ? nfs4_client_to_reclaim+0xe9/0x260 [ T1205] __kasan_report.cold+0x34/0x84 [ T1205] ? nfs4_client_to_reclaim+0xe9/0x260 [ T1205] kasan_report+0x3a/0x50 [ T1205] nfs4_client_to_reclaim+0xe9/0x260 [ T1205] ? nfsd4_release_lockowner+0x410/0x410 [ T1205] cld_pipe_downcall+0x5ca/0x760 [ T1205] ? nfsd4_cld_tracking_exit+0x1d0/0x1d0 [ T1205] ? down_write_killable_nested+0x170/0x170 [ T1205] ? avc_policy_seqno+0x28/0x40 [ T1205] ? selinux_file_permission+0x1b4/0x1e0 [ T1205] rpc_pipe_write+0x84/0xb0 [ T1205] vfs_write+0x143/0x520 [ T1205] ksys_write+0xc9/0x170 [ T1205] ? __ia32_sys_read+0x50/0x50 [ T1205] ? ktime_get_coarse_real_ts64+0xfe/0x110 [ T1205] ? ktime_get_coarse_real_ts64+0xa2/0x110 [ T1205] do_syscall_64+0x33/0x40 [ T1205] entry_SYSCALL_64_after_hwframe+0x67/0xd1 [ T1205] RIP: 0033:0x7fdbdb761bc7 [ T1205] Code: 0f 00 f7 d8 64 89 02 48 c7 c0 ff ff ff ff eb b7 0f 1f 00 f3 0f 1e fa 64 8b 04 25 18 00 00 00 85 c0 75 10 b8 01 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 514 [ T1205] RSP: 002b:00007fff8c4b7248 EFLAGS: 00000246 ORIG_RAX: 0000000000000001 [ T1205] RAX: ffffffffffffffda RBX: 000000000000042b RCX: 00007fdbdb761bc7 [ T1205] RDX: 000000000000042b RSI: 00007fff8c4b75f0 RDI: 0000000000000008 [ T1205] RBP: 00007fdbdb761bb0 R08: 0000000000000000 R09: 0000000000000001 [ T1205] R10: 0000000000000000 R11: 0000000000000246 R12: 000000000000042b [ T1205] R13: 0000000000000008 R14: 00007fff8c4b75f0 R15: 0000000000000000 [ T1205] ================================================================== Fix it by checking namelen. Signed-off-by: Li Lingfeng Fixes: 74725959c33c ("nfsd: un-deprecate nfsdcld") Reviewed-by: Jeff Layton Reviewed-by: Scott Mayhew Tested-by: Scott Mayhew Signed-off-by: Chuck Lever --- fs/nfsd/nfs4recover.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/fs/nfsd/nfs4recover.c b/fs/nfsd/nfs4recover.c index 0d25ad5eeae3..530686f32e9e 100644 --- a/fs/nfsd/nfs4recover.c +++ b/fs/nfsd/nfs4recover.c @@ -809,6 +809,10 @@ __cld_pipe_inprogress_downcall(const struct cld_msg_v2 __user *cmsg, ci = &cmsg->cm_u.cm_clntinfo; if (get_user(namelen, &ci->cc_name.cn_len)) return -EFAULT; + if (!namelen) { + dprintk("%s: namelen should not be zero", __func__); + return -EINVAL; + } name.data = memdup_user(&ci->cc_name.cn_id, namelen); if (IS_ERR(name.data)) return PTR_ERR(name.data); @@ -831,6 +835,10 @@ __cld_pipe_inprogress_downcall(const struct cld_msg_v2 __user *cmsg, cnm = &cmsg->cm_u.cm_name; if (get_user(namelen, &cnm->cn_len)) return -EFAULT; + if (!namelen) { + dprintk("%s: namelen should not be zero", __func__); + return -EINVAL; + } name.data = memdup_user(&cnm->cn_id, namelen); if (IS_ERR(name.data)) return PTR_ERR(name.data); From 5559c157b79907a901578f93f83eb6732bfcbc1a Mon Sep 17 00:00:00 2001 From: Scott Mayhew Date: Mon, 9 Sep 2024 16:28:54 -0400 Subject: [PATCH 257/302] nfsd: enforce upper limit for namelen in __cld_pipe_inprogress_downcall() This patch is intended to go on top of "nfsd: return -EINVAL when namelen is 0" from Li Lingfeng. Li's patch checks for 0, but we should be enforcing an upper bound as well. Note that if nfsdcld somehow gets an id > NFS4_OPAQUE_LIMIT in its database, it'll truncate it to NFS4_OPAQUE_LIMIT when it does the downcall anyway. Signed-off-by: Scott Mayhew Signed-off-by: Chuck Lever --- fs/nfsd/nfs4recover.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fs/nfsd/nfs4recover.c b/fs/nfsd/nfs4recover.c index 530686f32e9e..b7d61eb8afe9 100644 --- a/fs/nfsd/nfs4recover.c +++ b/fs/nfsd/nfs4recover.c @@ -809,8 +809,8 @@ __cld_pipe_inprogress_downcall(const struct cld_msg_v2 __user *cmsg, ci = &cmsg->cm_u.cm_clntinfo; if (get_user(namelen, &ci->cc_name.cn_len)) return -EFAULT; - if (!namelen) { - dprintk("%s: namelen should not be zero", __func__); + if (namelen == 0 || namelen > NFS4_OPAQUE_LIMIT) { + dprintk("%s: invalid namelen (%u)", __func__, namelen); return -EINVAL; } name.data = memdup_user(&ci->cc_name.cn_id, namelen); @@ -835,8 +835,8 @@ __cld_pipe_inprogress_downcall(const struct cld_msg_v2 __user *cmsg, cnm = &cmsg->cm_u.cm_name; if (get_user(namelen, &cnm->cn_len)) return -EFAULT; - if (!namelen) { - dprintk("%s: namelen should not be zero", __func__); + if (namelen == 0 || namelen > NFS4_OPAQUE_LIMIT) { + dprintk("%s: invalid namelen (%u)", __func__, namelen); return -EINVAL; } name.data = memdup_user(&cnm->cn_id, namelen); From a078a7dc0eaa9db288ae45319f7f7503968af546 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Thu, 29 Aug 2024 09:26:40 -0400 Subject: [PATCH 258/302] nfsd: untangle code in nfsd4_deleg_getattr_conflict() The code in nfsd4_deleg_getattr_conflict() is convoluted and buggy. With this patch we: - properly handle non-nfsd leases. We must not assume flc_owner is a delegation unless fl_lmops == &nfsd_lease_mng_ops - move the main code out of the for loop - have a single exit which calls nfs4_put_stid() (and other exits which don't need to call that) [ jlayton: refactored on top of Neil's other patch: nfsd: fix nfsd4_deleg_getattr_conflict in presence of third party lease ] Fixes: c5967721e106 ("NFSD: handle GETATTR conflict with write delegation") Signed-off-by: NeilBrown Signed-off-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/nfs4state.c | 137 +++++++++++++++++++++----------------------- 1 file changed, 65 insertions(+), 72 deletions(-) diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index a49aa75bc0b7..df69dc6af467 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -8834,6 +8834,7 @@ nfsd4_deleg_getattr_conflict(struct svc_rqst *rqstp, struct dentry *dentry, __be32 status; struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); struct file_lock_context *ctx; + struct nfs4_delegation *dp = NULL; struct file_lease *fl; struct iattr attrs; struct nfs4_cb_fattr *ncf; @@ -8843,84 +8844,76 @@ nfsd4_deleg_getattr_conflict(struct svc_rqst *rqstp, struct dentry *dentry, ctx = locks_inode_context(inode); if (!ctx) return 0; + +#define NON_NFSD_LEASE ((void *)1) + spin_lock(&ctx->flc_lock); for_each_file_lock(fl, &ctx->flc_lease) { - unsigned char type = fl->c.flc_type; - if (fl->c.flc_flags == FL_LAYOUT) continue; - if (fl->fl_lmops != &nfsd_lease_mng_ops) { - /* - * non-nfs lease, if it's a lease with F_RDLCK then - * we are done; there isn't any write delegation - * on this inode - */ - if (type == F_RDLCK) - break; - - nfsd_stats_wdeleg_getattr_inc(nn); - spin_unlock(&ctx->flc_lock); - - status = nfserrno(nfsd_open_break_lease(inode, NFSD_MAY_READ)); - if (status != nfserr_jukebox || - !nfsd_wait_for_delegreturn(rqstp, inode)) - return status; - return 0; - } - if (type == F_WRLCK) { - struct nfs4_delegation *dp = fl->c.flc_owner; - - if (dp->dl_recall.cb_clp == *(rqstp->rq_lease_breaker)) { - spin_unlock(&ctx->flc_lock); - return 0; - } - nfsd_stats_wdeleg_getattr_inc(nn); - dp = fl->c.flc_owner; - refcount_inc(&dp->dl_stid.sc_count); - ncf = &dp->dl_cb_fattr; - nfs4_cb_getattr(&dp->dl_cb_fattr); - spin_unlock(&ctx->flc_lock); - wait_on_bit_timeout(&ncf->ncf_cb_flags, CB_GETATTR_BUSY, - TASK_INTERRUPTIBLE, NFSD_CB_GETATTR_TIMEOUT); - if (ncf->ncf_cb_status) { - /* Recall delegation only if client didn't respond */ - status = nfserrno(nfsd_open_break_lease(inode, NFSD_MAY_READ)); - if (status != nfserr_jukebox || - !nfsd_wait_for_delegreturn(rqstp, inode)) { - nfs4_put_stid(&dp->dl_stid); - return status; - } - } - if (!ncf->ncf_file_modified && - (ncf->ncf_initial_cinfo != ncf->ncf_cb_change || - ncf->ncf_cur_fsize != ncf->ncf_cb_fsize)) - ncf->ncf_file_modified = true; - if (ncf->ncf_file_modified) { - int err; - - /* - * Per section 10.4.3 of RFC 8881, the server would - * not update the file's metadata with the client's - * modified size - */ - attrs.ia_mtime = attrs.ia_ctime = current_time(inode); - attrs.ia_valid = ATTR_MTIME | ATTR_CTIME | ATTR_DELEG; - inode_lock(inode); - err = notify_change(&nop_mnt_idmap, dentry, &attrs, NULL); - inode_unlock(inode); - if (err) { - nfs4_put_stid(&dp->dl_stid); - return nfserrno(err); - } - ncf->ncf_cur_fsize = ncf->ncf_cb_fsize; - *size = ncf->ncf_cur_fsize; - *modified = true; - } - nfs4_put_stid(&dp->dl_stid); - return 0; + if (fl->c.flc_type == F_WRLCK) { + if (fl->fl_lmops == &nfsd_lease_mng_ops) + dp = fl->c.flc_owner; + else + dp = NON_NFSD_LEASE; } break; } + if (dp == NULL || dp == NON_NFSD_LEASE || + dp->dl_recall.cb_clp == *(rqstp->rq_lease_breaker)) { + spin_unlock(&ctx->flc_lock); + if (dp == NON_NFSD_LEASE) { + status = nfserrno(nfsd_open_break_lease(inode, + NFSD_MAY_READ)); + if (status != nfserr_jukebox || + !nfsd_wait_for_delegreturn(rqstp, inode)) + return status; + } + return 0; + } + + nfsd_stats_wdeleg_getattr_inc(nn); + refcount_inc(&dp->dl_stid.sc_count); + ncf = &dp->dl_cb_fattr; + nfs4_cb_getattr(&dp->dl_cb_fattr); spin_unlock(&ctx->flc_lock); - return 0; + + wait_on_bit_timeout(&ncf->ncf_cb_flags, CB_GETATTR_BUSY, + TASK_INTERRUPTIBLE, NFSD_CB_GETATTR_TIMEOUT); + if (ncf->ncf_cb_status) { + /* Recall delegation only if client didn't respond */ + status = nfserrno(nfsd_open_break_lease(inode, NFSD_MAY_READ)); + if (status != nfserr_jukebox || + !nfsd_wait_for_delegreturn(rqstp, inode)) + goto out_status; + } + if (!ncf->ncf_file_modified && + (ncf->ncf_initial_cinfo != ncf->ncf_cb_change || + ncf->ncf_cur_fsize != ncf->ncf_cb_fsize)) + ncf->ncf_file_modified = true; + if (ncf->ncf_file_modified) { + int err; + + /* + * Per section 10.4.3 of RFC 8881, the server would + * not update the file's metadata with the client's + * modified size + */ + attrs.ia_mtime = attrs.ia_ctime = current_time(inode); + attrs.ia_valid = ATTR_MTIME | ATTR_CTIME | ATTR_DELEG; + inode_lock(inode); + err = notify_change(&nop_mnt_idmap, dentry, &attrs, NULL); + inode_unlock(inode); + if (err) { + status = nfserrno(err); + goto out_status; + } + ncf->ncf_cur_fsize = ncf->ncf_cb_fsize; + *size = ncf->ncf_cur_fsize; + *modified = true; + } + status = 0; +out_status: + nfs4_put_stid(&dp->dl_stid); + return status; } From bf92e5008b17f935a6de8b708551e02c2294121c Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Mon, 9 Sep 2024 10:40:53 -0400 Subject: [PATCH 259/302] nfsd: fix initial getattr on write delegation At this point in compound processing, currentfh refers to the parent of the file, not the file itself. Get the correct dentry from the delegation stateid instead. Fixes: c5967721e106 ("NFSD: handle GETATTR conflict with write delegation") Signed-off-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/nfs4state.c | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index df69dc6af467..cb5a9ab451c5 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -5914,6 +5914,28 @@ static void nfsd4_open_deleg_none_ext(struct nfsd4_open *open, int status) } } +static bool +nfs4_delegation_stat(struct nfs4_delegation *dp, struct svc_fh *currentfh, + struct kstat *stat) +{ + struct nfsd_file *nf = find_rw_file(dp->dl_stid.sc_file); + struct path path; + int rc; + + if (!nf) + return false; + + path.mnt = currentfh->fh_export->ex_path.mnt; + path.dentry = file_dentry(nf->nf_file); + + rc = vfs_getattr(&path, stat, + (STATX_SIZE | STATX_CTIME | STATX_CHANGE_COOKIE), + AT_STATX_SYNC_AS_STAT); + + nfsd_file_put(nf); + return rc == 0; +} + /* * The Linux NFS server does not offer write delegations to NFSv4.0 * clients in order to avoid conflicts between write delegations and @@ -5949,7 +5971,6 @@ nfs4_open_delegation(struct nfsd4_open *open, struct nfs4_ol_stateid *stp, int cb_up; int status = 0; struct kstat stat; - struct path path; cb_up = nfsd4_cb_channel_good(oo->oo_owner.so_client); open->op_recall = false; @@ -5985,20 +6006,16 @@ nfs4_open_delegation(struct nfsd4_open *open, struct nfs4_ol_stateid *stp, memcpy(&open->op_delegate_stateid, &dp->dl_stid.sc_stateid, sizeof(dp->dl_stid.sc_stateid)); if (open->op_share_access & NFS4_SHARE_ACCESS_WRITE) { - open->op_delegate_type = NFS4_OPEN_DELEGATE_WRITE; - trace_nfsd_deleg_write(&dp->dl_stid.sc_stateid); - path.mnt = currentfh->fh_export->ex_path.mnt; - path.dentry = currentfh->fh_dentry; - if (vfs_getattr(&path, &stat, - (STATX_SIZE | STATX_CTIME | STATX_CHANGE_COOKIE), - AT_STATX_SYNC_AS_STAT)) { + if (!nfs4_delegation_stat(dp, currentfh, &stat)) { nfs4_put_stid(&dp->dl_stid); destroy_delegation(dp); goto out_no_deleg; } + open->op_delegate_type = NFS4_OPEN_DELEGATE_WRITE; dp->dl_cb_fattr.ncf_cur_fsize = stat.size; dp->dl_cb_fattr.ncf_initial_cinfo = nfsd4_change_attribute(&stat, d_inode(currentfh->fh_dentry)); + trace_nfsd_deleg_write(&dp->dl_stid.sc_stateid); } else { open->op_delegate_type = NFS4_OPEN_DELEGATE_READ; trace_nfsd_deleg_read(&dp->dl_stid.sc_stateid); From 45bb63ed20e02ae146336412889fe5450316a84f Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Mon, 9 Sep 2024 15:06:36 +1000 Subject: [PATCH 260/302] nfsd: fix delegation_blocked() to block correctly for at least 30 seconds The pair of bloom filtered used by delegation_blocked() was intended to block delegations on given filehandles for between 30 and 60 seconds. A new filehandle would be recorded in the "new" bit set. That would then be switch to the "old" bit set between 0 and 30 seconds later, and it would remain as the "old" bit set for 30 seconds. Unfortunately the code intended to clear the old bit set once it reached 30 seconds old, preparing it to be the next new bit set, instead cleared the *new* bit set before switching it to be the old bit set. This means that the "old" bit set is always empty and delegations are blocked between 0 and 30 seconds. This patch updates bd->new before clearing the set with that index, instead of afterwards. Reported-by: Olga Kornievskaia Cc: stable@vger.kernel.org Fixes: 6282cd565553 ("NFSD: Don't hand out delegations for 30 seconds after recalling them.") Signed-off-by: NeilBrown Reviewed-by: Benjamin Coddington Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/nfs4state.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index cb5a9ab451c5..ac1859c7cc9d 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -1078,7 +1078,8 @@ static void nfs4_free_deleg(struct nfs4_stid *stid) * When a delegation is recalled, the filehandle is stored in the "new" * filter. * Every 30 seconds we swap the filters and clear the "new" one, - * unless both are empty of course. + * unless both are empty of course. This results in delegations for a + * given filehandle being blocked for between 30 and 60 seconds. * * Each filter is 256 bits. We hash the filehandle to 32bit and use the * low 3 bytes as hash-table indices. @@ -1107,9 +1108,9 @@ static int delegation_blocked(struct knfsd_fh *fh) if (ktime_get_seconds() - bd->swap_time > 30) { bd->entries -= bd->old_entries; bd->old_entries = bd->entries; + bd->new = 1-bd->new; memset(bd->set[bd->new], 0, sizeof(bd->set[0])); - bd->new = 1-bd->new; bd->swap_time = ktime_get_seconds(); } spin_unlock(&blocked_delegations_lock); From 4b132aacb0768ac1e652cf517097ea6f237214b9 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Fri, 13 Sep 2024 14:08:13 -0400 Subject: [PATCH 261/302] tools: Add xdrgen Add a Python-based tool for translating XDR specifications into XDR encoder and decoder functions written in the Linux kernel's C coding style. The generator attempts to match the usual C coding style of the Linux kernel's SunRPC consumers. This approach is similar to the netlink code generator in tools/net/ynl . The maintainability benefits of machine-generated XDR code include: - Stronger type checking - Reduces the number of bugs introduced by human error - Makes the XDR code easier to audit and analyze - Enables rapid prototyping of new RPC-based protocols - Hardens the layering between protocol logic and marshaling - Makes it easier to add observability on demand - Unit tests might be built for both the tool and (automatically) for the generated code In addition, converting the XDR layer to use memory-safe languages such as Rust will be easier if much of the code can be converted automatically. Tested-by: Jeff Layton Signed-off-by: Chuck Lever --- include/linux/sunrpc/xdrgen/_builtins.h | 243 +++++++++ include/linux/sunrpc/xdrgen/_defs.h | 26 + tools/net/sunrpc/xdrgen/.gitignore | 2 + tools/net/sunrpc/xdrgen/README | 244 +++++++++ tools/net/sunrpc/xdrgen/__init__.py | 2 + .../net/sunrpc/xdrgen/generators/__init__.py | 113 ++++ .../net/sunrpc/xdrgen/generators/constant.py | 20 + tools/net/sunrpc/xdrgen/generators/enum.py | 44 ++ .../sunrpc/xdrgen/generators/header_bottom.py | 33 ++ .../sunrpc/xdrgen/generators/header_top.py | 45 ++ tools/net/sunrpc/xdrgen/generators/pointer.py | 272 ++++++++++ tools/net/sunrpc/xdrgen/generators/program.py | 168 ++++++ .../sunrpc/xdrgen/generators/source_top.py | 32 ++ tools/net/sunrpc/xdrgen/generators/struct.py | 272 ++++++++++ tools/net/sunrpc/xdrgen/generators/typedef.py | 255 +++++++++ tools/net/sunrpc/xdrgen/generators/union.py | 243 +++++++++ tools/net/sunrpc/xdrgen/grammars/xdr.lark | 119 ++++ tools/net/sunrpc/xdrgen/subcmds/__init__.py | 2 + .../net/sunrpc/xdrgen/subcmds/declarations.py | 76 +++ .../net/sunrpc/xdrgen/subcmds/definitions.py | 78 +++ tools/net/sunrpc/xdrgen/subcmds/lint.py | 33 ++ tools/net/sunrpc/xdrgen/subcmds/source.py | 118 ++++ .../templates/C/constants/definition.j2 | 3 + .../templates/C/enum/declaration/close.j2 | 4 + .../xdrgen/templates/C/enum/decoder/enum.j2 | 19 + .../templates/C/enum/definition/close.j2 | 2 + .../templates/C/enum/definition/enumerator.j2 | 2 + .../templates/C/enum/definition/open.j2 | 3 + .../xdrgen/templates/C/enum/encoder/enum.j2 | 14 + .../C/header_bottom/declaration/header.j2 | 3 + .../C/header_bottom/definition/header.j2 | 3 + .../C/header_top/declaration/header.j2 | 14 + .../C/header_top/definition/header.j2 | 10 + .../templates/C/pointer/declaration/close.j2 | 4 + .../templates/C/pointer/decoder/basic.j2 | 6 + .../templates/C/pointer/decoder/close.j2 | 3 + .../C/pointer/decoder/fixed_length_array.j2 | 8 + .../C/pointer/decoder/fixed_length_opaque.j2 | 6 + .../templates/C/pointer/decoder/open.j2 | 22 + .../C/pointer/decoder/optional_data.j2 | 6 + .../pointer/decoder/variable_length_array.j2 | 13 + .../pointer/decoder/variable_length_opaque.j2 | 6 + .../pointer/decoder/variable_length_string.j2 | 6 + .../templates/C/pointer/definition/basic.j2 | 5 + .../templates/C/pointer/definition/close.j2 | 2 + .../pointer/definition/fixed_length_array.j2 | 5 + .../pointer/definition/fixed_length_opaque.j2 | 5 + .../templates/C/pointer/definition/open.j2 | 6 + .../C/pointer/definition/optional_data.j2 | 5 + .../definition/variable_length_array.j2 | 8 + .../definition/variable_length_opaque.j2 | 5 + .../definition/variable_length_string.j2 | 5 + .../templates/C/pointer/encoder/basic.j2 | 10 + .../templates/C/pointer/encoder/close.j2 | 3 + .../C/pointer/encoder/fixed_length_array.j2 | 12 + .../C/pointer/encoder/fixed_length_opaque.j2 | 6 + .../templates/C/pointer/encoder/open.j2 | 20 + .../C/pointer/encoder/optional_data.j2 | 6 + .../pointer/encoder/variable_length_array.j2 | 15 + .../pointer/encoder/variable_length_opaque.j2 | 8 + .../pointer/encoder/variable_length_string.j2 | 8 + .../C/program/declaration/argument.j2 | 2 + .../templates/C/program/declaration/result.j2 | 2 + .../templates/C/program/decoder/argument.j2 | 21 + .../templates/C/program/decoder/result.j2 | 22 + .../templates/C/program/definition/close.j2 | 2 + .../templates/C/program/definition/open.j2 | 6 + .../C/program/definition/procedure.j2 | 2 + .../templates/C/program/encoder/argument.j2 | 16 + .../templates/C/program/encoder/result.j2 | 21 + .../xdrgen/templates/C/source_top/client.j2 | 8 + .../xdrgen/templates/C/source_top/server.j2 | 8 + .../templates/C/struct/declaration/close.j2 | 4 + .../templates/C/struct/decoder/basic.j2 | 6 + .../templates/C/struct/decoder/close.j2 | 3 + .../C/struct/decoder/fixed_length_array.j2 | 8 + .../C/struct/decoder/fixed_length_opaque.j2 | 6 + .../xdrgen/templates/C/struct/decoder/open.j2 | 12 + .../C/struct/decoder/optional_data.j2 | 6 + .../C/struct/decoder/variable_length_array.j2 | 13 + .../struct/decoder/variable_length_opaque.j2 | 6 + .../struct/decoder/variable_length_string.j2 | 6 + .../templates/C/struct/definition/basic.j2 | 5 + .../templates/C/struct/definition/close.j2 | 2 + .../C/struct/definition/fixed_length_array.j2 | 5 + .../struct/definition/fixed_length_opaque.j2 | 5 + .../templates/C/struct/definition/open.j2 | 6 + .../C/struct/definition/optional_data.j2 | 5 + .../definition/variable_length_array.j2 | 8 + .../definition/variable_length_opaque.j2 | 5 + .../definition/variable_length_string.j2 | 5 + .../templates/C/struct/encoder/basic.j2 | 10 + .../templates/C/struct/encoder/close.j2 | 3 + .../C/struct/encoder/fixed_length_array.j2 | 12 + .../C/struct/encoder/fixed_length_opaque.j2 | 6 + .../xdrgen/templates/C/struct/encoder/open.j2 | 12 + .../C/struct/encoder/optional_data.j2 | 6 + .../C/struct/encoder/variable_length_array.j2 | 15 + .../struct/encoder/variable_length_opaque.j2 | 8 + .../struct/encoder/variable_length_string.j2 | 8 + .../templates/C/typedef/declaration/basic.j2 | 8 + .../typedef/declaration/fixed_length_array.j2 | 4 + .../declaration/fixed_length_opaque.j2 | 4 + .../declaration/variable_length_array.j2 | 4 + .../declaration/variable_length_opaque.j2 | 4 + .../declaration/variable_length_string.j2 | 4 + .../templates/C/typedef/decoder/basic.j2 | 17 + .../C/typedef/decoder/fixed_length_array.j2 | 25 + .../C/typedef/decoder/fixed_length_opaque.j2 | 17 + .../typedef/decoder/variable_length_array.j2 | 26 + .../typedef/decoder/variable_length_opaque.j2 | 17 + .../typedef/decoder/variable_length_string.j2 | 17 + .../templates/C/typedef/definition/basic.j2 | 6 + .../typedef/definition/fixed_length_array.j2 | 6 + .../typedef/definition/fixed_length_opaque.j2 | 6 + .../definition/variable_length_array.j2 | 9 + .../definition/variable_length_opaque.j2 | 6 + .../definition/variable_length_string.j2 | 6 + .../templates/C/typedef/encoder/basic.j2 | 21 + .../C/typedef/encoder/fixed_length_array.j2 | 25 + .../C/typedef/encoder/fixed_length_opaque.j2 | 17 + .../typedef/encoder/variable_length_array.j2 | 30 ++ .../typedef/encoder/variable_length_opaque.j2 | 17 + .../typedef/encoder/variable_length_string.j2 | 17 + .../xdrgen/templates/C/union/decoder/basic.j2 | 6 + .../xdrgen/templates/C/union/decoder/break.j2 | 2 + .../templates/C/union/decoder/case_spec.j2 | 2 + .../xdrgen/templates/C/union/decoder/close.j2 | 4 + .../templates/C/union/decoder/default_spec.j2 | 2 + .../xdrgen/templates/C/union/decoder/open.j2 | 12 + .../C/union/decoder/optional_data.j2 | 6 + .../templates/C/union/decoder/switch_spec.j2 | 7 + .../C/union/decoder/variable_length_array.j2 | 13 + .../C/union/decoder/variable_length_opaque.j2 | 6 + .../C/union/decoder/variable_length_string.j2 | 6 + .../xdrgen/templates/C/union/decoder/void.j2 | 3 + .../templates/C/union/definition/case_spec.j2 | 2 + .../templates/C/union/definition/close.j2 | 8 + .../C/union/definition/default_spec.j2 | 2 + .../templates/C/union/definition/open.j2 | 6 + .../C/union/definition/switch_spec.j2 | 3 + .../xdrgen/templates/C/union/encoder/basic.j2 | 10 + .../xdrgen/templates/C/union/encoder/break.j2 | 2 + .../templates/C/union/encoder/case_spec.j2 | 2 + .../xdrgen/templates/C/union/encoder/close.j2 | 4 + .../templates/C/union/encoder/default_spec.j2 | 2 + .../xdrgen/templates/C/union/encoder/open.j2 | 12 + .../templates/C/union/encoder/switch_spec.j2 | 7 + .../xdrgen/templates/C/union/encoder/void.j2 | 3 + tools/net/sunrpc/xdrgen/tests/test.x | 36 ++ tools/net/sunrpc/xdrgen/xdr_ast.py | 510 ++++++++++++++++++ tools/net/sunrpc/xdrgen/xdr_parse.py | 36 ++ tools/net/sunrpc/xdrgen/xdrgen | 132 +++++ 153 files changed, 4196 insertions(+) create mode 100644 include/linux/sunrpc/xdrgen/_builtins.h create mode 100644 include/linux/sunrpc/xdrgen/_defs.h create mode 100644 tools/net/sunrpc/xdrgen/.gitignore create mode 100644 tools/net/sunrpc/xdrgen/README create mode 100644 tools/net/sunrpc/xdrgen/__init__.py create mode 100644 tools/net/sunrpc/xdrgen/generators/__init__.py create mode 100644 tools/net/sunrpc/xdrgen/generators/constant.py create mode 100644 tools/net/sunrpc/xdrgen/generators/enum.py create mode 100644 tools/net/sunrpc/xdrgen/generators/header_bottom.py create mode 100644 tools/net/sunrpc/xdrgen/generators/header_top.py create mode 100644 tools/net/sunrpc/xdrgen/generators/pointer.py create mode 100644 tools/net/sunrpc/xdrgen/generators/program.py create mode 100644 tools/net/sunrpc/xdrgen/generators/source_top.py create mode 100644 tools/net/sunrpc/xdrgen/generators/struct.py create mode 100644 tools/net/sunrpc/xdrgen/generators/typedef.py create mode 100644 tools/net/sunrpc/xdrgen/generators/union.py create mode 100644 tools/net/sunrpc/xdrgen/grammars/xdr.lark create mode 100644 tools/net/sunrpc/xdrgen/subcmds/__init__.py create mode 100644 tools/net/sunrpc/xdrgen/subcmds/declarations.py create mode 100644 tools/net/sunrpc/xdrgen/subcmds/definitions.py create mode 100644 tools/net/sunrpc/xdrgen/subcmds/lint.py create mode 100644 tools/net/sunrpc/xdrgen/subcmds/source.py create mode 100644 tools/net/sunrpc/xdrgen/templates/C/constants/definition.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/enum/declaration/close.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/enum/decoder/enum.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/enum/definition/close.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/enum/definition/enumerator.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/enum/definition/open.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/enum/encoder/enum.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/header_bottom/declaration/header.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/header_bottom/definition/header.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/header_top/declaration/header.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/header_top/definition/header.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/pointer/declaration/close.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/basic.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/close.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/fixed_length_array.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/fixed_length_opaque.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/open.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/optional_data.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/variable_length_array.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/variable_length_opaque.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/variable_length_string.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/pointer/definition/basic.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/pointer/definition/close.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/pointer/definition/fixed_length_array.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/pointer/definition/fixed_length_opaque.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/pointer/definition/open.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/pointer/definition/optional_data.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/pointer/definition/variable_length_array.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/pointer/definition/variable_length_opaque.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/pointer/definition/variable_length_string.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/basic.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/close.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/fixed_length_array.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/fixed_length_opaque.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/open.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/optional_data.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/variable_length_array.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/variable_length_opaque.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/variable_length_string.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/program/declaration/argument.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/program/declaration/result.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/program/decoder/argument.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/program/decoder/result.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/program/definition/close.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/program/definition/open.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/program/definition/procedure.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/program/encoder/argument.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/program/encoder/result.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/source_top/client.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/source_top/server.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/struct/declaration/close.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/struct/decoder/basic.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/struct/decoder/close.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/struct/decoder/fixed_length_array.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/struct/decoder/fixed_length_opaque.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/struct/decoder/open.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/struct/decoder/optional_data.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/struct/decoder/variable_length_array.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/struct/decoder/variable_length_opaque.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/struct/decoder/variable_length_string.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/struct/definition/basic.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/struct/definition/close.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/struct/definition/fixed_length_array.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/struct/definition/fixed_length_opaque.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/struct/definition/open.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/struct/definition/optional_data.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/struct/definition/variable_length_array.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/struct/definition/variable_length_opaque.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/struct/definition/variable_length_string.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/struct/encoder/basic.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/struct/encoder/close.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/struct/encoder/fixed_length_array.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/struct/encoder/fixed_length_opaque.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/struct/encoder/open.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/struct/encoder/optional_data.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/struct/encoder/variable_length_array.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/struct/encoder/variable_length_opaque.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/struct/encoder/variable_length_string.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/typedef/declaration/basic.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/typedef/declaration/fixed_length_array.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/typedef/declaration/fixed_length_opaque.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/typedef/declaration/variable_length_array.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/typedef/declaration/variable_length_opaque.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/typedef/declaration/variable_length_string.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/basic.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/fixed_length_array.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/fixed_length_opaque.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/variable_length_array.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/variable_length_opaque.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/variable_length_string.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/typedef/definition/basic.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/typedef/definition/fixed_length_array.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/typedef/definition/fixed_length_opaque.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/typedef/definition/variable_length_array.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/typedef/definition/variable_length_opaque.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/typedef/definition/variable_length_string.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/basic.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/fixed_length_array.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/fixed_length_opaque.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/variable_length_array.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/variable_length_opaque.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/variable_length_string.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/union/decoder/basic.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/union/decoder/break.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/union/decoder/case_spec.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/union/decoder/close.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/union/decoder/default_spec.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/union/decoder/open.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/union/decoder/optional_data.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/union/decoder/switch_spec.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/union/decoder/variable_length_array.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/union/decoder/variable_length_opaque.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/union/decoder/variable_length_string.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/union/decoder/void.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/union/definition/case_spec.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/union/definition/close.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/union/definition/default_spec.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/union/definition/open.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/union/definition/switch_spec.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/union/encoder/basic.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/union/encoder/break.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/union/encoder/case_spec.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/union/encoder/close.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/union/encoder/default_spec.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/union/encoder/open.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/union/encoder/switch_spec.j2 create mode 100644 tools/net/sunrpc/xdrgen/templates/C/union/encoder/void.j2 create mode 100644 tools/net/sunrpc/xdrgen/tests/test.x create mode 100644 tools/net/sunrpc/xdrgen/xdr_ast.py create mode 100644 tools/net/sunrpc/xdrgen/xdr_parse.py create mode 100755 tools/net/sunrpc/xdrgen/xdrgen diff --git a/include/linux/sunrpc/xdrgen/_builtins.h b/include/linux/sunrpc/xdrgen/_builtins.h new file mode 100644 index 000000000000..68746c59fc9a --- /dev/null +++ b/include/linux/sunrpc/xdrgen/_builtins.h @@ -0,0 +1,243 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2024 Oracle and/or its affiliates. + * + * This header defines XDR data type primitives specified in + * Section 4 of RFC 4506, used by RPC programs implemented + * in the Linux kernel. + */ + +#ifndef _SUNRPC_XDRGEN__BUILTINS_H_ +#define _SUNRPC_XDRGEN__BUILTINS_H_ + +#include + +static inline bool +xdrgen_decode_void(struct xdr_stream *xdr) +{ + return true; +} + +static inline bool +xdrgen_encode_void(struct xdr_stream *xdr) +{ + return true; +} + +static inline bool +xdrgen_decode_bool(struct xdr_stream *xdr, bool *ptr) +{ + __be32 *p = xdr_inline_decode(xdr, XDR_UNIT); + + if (unlikely(!p)) + return false; + *ptr = (*p != xdr_zero); + return true; +} + +static inline bool +xdrgen_encode_bool(struct xdr_stream *xdr, bool val) +{ + __be32 *p = xdr_reserve_space(xdr, XDR_UNIT); + + if (unlikely(!p)) + return false; + *p = val ? xdr_one : xdr_zero; + return true; +} + +static inline bool +xdrgen_decode_int(struct xdr_stream *xdr, s32 *ptr) +{ + __be32 *p = xdr_inline_decode(xdr, XDR_UNIT); + + if (unlikely(!p)) + return false; + *ptr = be32_to_cpup(p); + return true; +} + +static inline bool +xdrgen_encode_int(struct xdr_stream *xdr, s32 val) +{ + __be32 *p = xdr_reserve_space(xdr, XDR_UNIT); + + if (unlikely(!p)) + return false; + *p = cpu_to_be32(val); + return true; +} + +static inline bool +xdrgen_decode_unsigned_int(struct xdr_stream *xdr, u32 *ptr) +{ + __be32 *p = xdr_inline_decode(xdr, XDR_UNIT); + + if (unlikely(!p)) + return false; + *ptr = be32_to_cpup(p); + return true; +} + +static inline bool +xdrgen_encode_unsigned_int(struct xdr_stream *xdr, u32 val) +{ + __be32 *p = xdr_reserve_space(xdr, XDR_UNIT); + + if (unlikely(!p)) + return false; + *p = cpu_to_be32(val); + return true; +} + +static inline bool +xdrgen_decode_long(struct xdr_stream *xdr, s32 *ptr) +{ + __be32 *p = xdr_inline_decode(xdr, XDR_UNIT); + + if (unlikely(!p)) + return false; + *ptr = be32_to_cpup(p); + return true; +} + +static inline bool +xdrgen_encode_long(struct xdr_stream *xdr, s32 val) +{ + __be32 *p = xdr_reserve_space(xdr, XDR_UNIT); + + if (unlikely(!p)) + return false; + *p = cpu_to_be32(val); + return true; +} + +static inline bool +xdrgen_decode_unsigned_long(struct xdr_stream *xdr, u32 *ptr) +{ + __be32 *p = xdr_inline_decode(xdr, XDR_UNIT); + + if (unlikely(!p)) + return false; + *ptr = be32_to_cpup(p); + return true; +} + +static inline bool +xdrgen_encode_unsigned_long(struct xdr_stream *xdr, u32 val) +{ + __be32 *p = xdr_reserve_space(xdr, XDR_UNIT); + + if (unlikely(!p)) + return false; + *p = cpu_to_be32(val); + return true; +} + +static inline bool +xdrgen_decode_hyper(struct xdr_stream *xdr, s64 *ptr) +{ + __be32 *p = xdr_inline_decode(xdr, XDR_UNIT * 2); + + if (unlikely(!p)) + return false; + *ptr = get_unaligned_be64(p); + return true; +} + +static inline bool +xdrgen_encode_hyper(struct xdr_stream *xdr, s64 val) +{ + __be32 *p = xdr_reserve_space(xdr, XDR_UNIT * 2); + + if (unlikely(!p)) + return false; + put_unaligned_be64(val, p); + return true; +} + +static inline bool +xdrgen_decode_unsigned_hyper(struct xdr_stream *xdr, u64 *ptr) +{ + __be32 *p = xdr_inline_decode(xdr, XDR_UNIT * 2); + + if (unlikely(!p)) + return false; + *ptr = get_unaligned_be64(p); + return true; +} + +static inline bool +xdrgen_encode_unsigned_hyper(struct xdr_stream *xdr, u64 val) +{ + __be32 *p = xdr_reserve_space(xdr, XDR_UNIT * 2); + + if (unlikely(!p)) + return false; + put_unaligned_be64(val, p); + return true; +} + +static inline bool +xdrgen_decode_string(struct xdr_stream *xdr, string *ptr, u32 maxlen) +{ + __be32 *p; + u32 len; + + if (unlikely(xdr_stream_decode_u32(xdr, &len) != XDR_UNIT)) + return false; + if (unlikely(maxlen && len > maxlen)) + return false; + if (len != 0) { + p = xdr_inline_decode(xdr, len); + if (unlikely(!p)) + return false; + ptr->data = (unsigned char *)p; + } + ptr->len = len; + return true; +} + +static inline bool +xdrgen_encode_string(struct xdr_stream *xdr, string val, u32 maxlen) +{ + __be32 *p = xdr_reserve_space(xdr, XDR_UNIT + xdr_align_size(val.len)); + + if (unlikely(!p)) + return false; + xdr_encode_opaque(p, val.data, val.len); + return true; +} + +static inline bool +xdrgen_decode_opaque(struct xdr_stream *xdr, opaque *ptr, u32 maxlen) +{ + __be32 *p; + u32 len; + + if (unlikely(xdr_stream_decode_u32(xdr, &len) != XDR_UNIT)) + return false; + if (unlikely(maxlen && len > maxlen)) + return false; + if (len != 0) { + p = xdr_inline_decode(xdr, len); + if (unlikely(!p)) + return false; + ptr->data = (u8 *)p; + } + ptr->len = len; + return true; +} + +static inline bool +xdrgen_encode_opaque(struct xdr_stream *xdr, opaque val) +{ + __be32 *p = xdr_reserve_space(xdr, XDR_UNIT + xdr_align_size(val.len)); + + if (unlikely(!p)) + return false; + xdr_encode_opaque(p, val.data, val.len); + return true; +} + +#endif /* _SUNRPC_XDRGEN__BUILTINS_H_ */ diff --git a/include/linux/sunrpc/xdrgen/_defs.h b/include/linux/sunrpc/xdrgen/_defs.h new file mode 100644 index 000000000000..be9e62371758 --- /dev/null +++ b/include/linux/sunrpc/xdrgen/_defs.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2024 Oracle and/or its affiliates. + * + * This header defines XDR data type primitives specified in + * Section 4 of RFC 4506, used by RPC programs implemented + * in the Linux kernel. + */ + +#ifndef _SUNRPC_XDRGEN__DEFS_H_ +#define _SUNRPC_XDRGEN__DEFS_H_ + +#define TRUE (true) +#define FALSE (false) + +typedef struct { + u32 len; + unsigned char *data; +} string; + +typedef struct { + u32 len; + u8 *data; +} opaque; + +#endif /* _SUNRPC_XDRGEN__DEFS_H_ */ diff --git a/tools/net/sunrpc/xdrgen/.gitignore b/tools/net/sunrpc/xdrgen/.gitignore new file mode 100644 index 000000000000..d7366c2f9be8 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/.gitignore @@ -0,0 +1,2 @@ +__pycache__ +generators/__pycache__ diff --git a/tools/net/sunrpc/xdrgen/README b/tools/net/sunrpc/xdrgen/README new file mode 100644 index 000000000000..92f7738ad50c --- /dev/null +++ b/tools/net/sunrpc/xdrgen/README @@ -0,0 +1,244 @@ +xdrgen - Linux Kernel XDR code generator + +Introduction +------------ + +SunRPC programs are typically specified using a language defined by +RFC 4506. In fact, all IETF-published NFS specifications provide a +description of the specified protocol using this language. + +Since the 1990's, user space consumers of SunRPC have had access to +a tool that could read such XDR specifications and then generate C +code that implements the RPC portions of that protocol. This tool is +called rpcgen. + +This RPC-level code is code that handles input directly from the +network, and thus a high degree of memory safety and sanity checking +is needed to help ensure proper levels of security. Bugs in this +code can have significant impact on security and performance. + +However, it is code that is repetitive and tedious to write by hand. + +The C code generated by rpcgen makes extensive use of the facilities +of the user space TI-RPC library and libc. Furthermore, the dialect +of the generated code is very traditional K&R C. + +The Linux kernel's implementation of SunRPC-based protocols hand-roll +their XDR implementation. There are two main reasons for this: + +1. libtirpc (and its predecessors) operate only in user space. The + kernel's RPC implementation and its API are significantly + different than libtirpc. + +2. rpcgen-generated code is believed to be less efficient than code + that is hand-written. + +These days, gcc and its kin are capable of optimizing code better +than human authors. There are only a few instances where writing +XDR code by hand will make a measurable performance different. + +In addition, the current hand-written code in the Linux kernel is +difficult to audit and prove that it implements exactly what is in +the protocol specification. + +In order to accrue the benefits of machine-generated XDR code in the +kernel, a tool is needed that will output C code that works against +the kernel's SunRPC implementation rather than libtirpc. + +Enter xdrgen. + + +Dependencies +------------ + +These dependencies are typically packaged by Linux distributions: + +- python3 +- python3-lark +- python3-jinja2 + +These dependencies are available via PyPi: + +- pip install 'lark[interegular]' + + +XDR Specifications +------------------ + +When adding a new protocol implementation to the kernel, the XDR +specification can be derived by feeding a .txt copy of the RFC to +the script located in tools/net/sunrpc/extract.sh. + + $ extract.sh < rfc0001.txt > new2.x + + +Operation +--------- + +Once a .x file is available, use xdrgen to generate source and +header files containing an implementation of XDR encoding and +decoding functions for the specified protocol. + + $ ./xdrgen definitions new2.x > include/linux/sunrpc/xdrgen/new2.h + $ ./xdrgen declarations new2.x > new2xdr_gen.h + +and + + $ ./xdrgen source new2.x > new2xdr_gen.c + +The files are ready to use for a server-side protocol implementation, +or may be used as a guide for implementing these routines by hand. + +By default, the only comments added to this code are kdoc comments +that appear directly in front of the public per-procedure APIs. For +deeper introspection, specifying the "--annotate" flag will insert +additional comments in the generated code to help readers match the +generated code to specific parts of the XDR specification. + +Because the generated code is targeted for the Linux kernel, it +is tagged with a GPLv2-only license. + +The xdrgen tool can also provide lexical and syntax checking of +an XDR specification: + + $ ./xdrgen lint xdr/new.x + + +How It Works +------------ + +xdrgen does not use machine learning to generate source code. The +translation is entirely deterministic. + +RFC 4506 Section 6 contains a BNF grammar of the XDR specification +language. The grammar has been adapted for use by the Python Lark +module. + +The xdr.ebnf file in this directory contains the grammar used to +parse XDR specifications. xdrgen configures Lark using the grammar +in xdr.ebnf. Lark parses the target XDR specification using this +grammar, creating a parse tree. + +xdrgen then transforms the parse tree into an abstract syntax tree. +This tree is passed to a series of code generators. + +The generators are implemented as Python classes residing in the +generators/ directory. Each generator emits code created from Jinja2 +templates stored in the templates/ directory. + +The source code is generated in the same order in which they appear +in the specification to ensure the generated code compiles. This +conforms with the behavior of rpcgen. + +xdrgen assumes that the generated source code is further compiled by +a compiler that can optimize in a number of ways, including: + + - Unused functions are discarded (ie, not added to the executable) + + - Aggressive function inlining removes unnecessary stack frames + + - Single-arm switch statements are replaced by a single conditional + branch + +And so on. + + +Pragmas +------- + +Pragma directives specify exceptions to the normal generation of +encoding and decoding functions. Currently one directive is +implemented: "public". + +Pragma exclude +------ ------- + + pragma exclude ; + +In some cases, a procedure encoder or decoder function might need +special processing that cannot be automatically generated. The +automatically-generated functions might conflict or interfere with +the hand-rolled function. To avoid editing the generated source code +by hand, a pragma can specify that the procedure's encoder and +decoder functions are not included in the generated header and +source. + +For example: + + pragma exclude NFSPROC3_READDIRPLUS; + +Excludes the decoder function for the READDIRPLUS argument and the +encoder function for the READDIRPLUS result. + +Note that because data item encoder and decoder functions are +defined "static __maybe_unused", subsequent compilation +automatically excludes data item encoder and decoder functions that +are used only by excluded procedure. + +Pragma header +------ ------ + + pragma header ; + +Provide a name to use for the header file. For example: + + pragma header nlm4; + +Adds + + #include "nlm4xdr_gen.h" + +to the generated source file. + +Pragma public +------ ------ + + pragma public ; + +Normally XDR encoder and decoder functions are "static". In case an +implementer wants to call these functions from other source code, +s/he can add a public pragma in the input .x file to indicate a set +of functions that should get a prototype in the generated header, +and the function definitions will not be declared static. + +For example: + + pragma public nfsstat3; + +Adds these prototypes in the generated header: + + bool xdrgen_decode_nfsstat3(struct xdr_stream *xdr, enum nfsstat3 *ptr); + bool xdrgen_encode_nfsstat3(struct xdr_stream *xdr, enum nfsstat3 value); + +And, in the generated source code, both of these functions appear +without the "static __maybe_unused" modifiers. + + +Future Work +----------- + +Finish implementing XDR pointer and list types. + +Generate client-side procedure functions + +Expand the README into a user guide similar to rpcgen(1) + +Add more pragma directives: + + * @pages -- use xdr_read/write_pages() for the specified opaque + field + * @skip -- do not decode, but rather skip, the specified argument + field + +Enable something like a #include to dynamically insert the content +of other specification files + +Properly support line-by-line pass-through via the "%" decorator + +Build a unit test suite for verifying translation of XDR language +into compilable code + +Add a command-line option to insert trace_printk call sites in the +generated source code, for improved (temporary) observability + +Generate kernel Rust code as well as C code diff --git a/tools/net/sunrpc/xdrgen/__init__.py b/tools/net/sunrpc/xdrgen/__init__.py new file mode 100644 index 000000000000..c940e9275252 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +# Just to make sphinx-apidoc document this directory diff --git a/tools/net/sunrpc/xdrgen/generators/__init__.py b/tools/net/sunrpc/xdrgen/generators/__init__.py new file mode 100644 index 000000000000..fd2457461274 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/generators/__init__.py @@ -0,0 +1,113 @@ +# SPDX-License-Identifier: GPL-2.0 + +"""Define a base code generator class""" + +import sys +from jinja2 import Environment, FileSystemLoader, Template + +from xdr_ast import _XdrAst, Specification, _RpcProgram, _XdrTypeSpecifier +from xdr_ast import public_apis, pass_by_reference, get_header_name +from xdr_parse import get_xdr_annotate + + +def create_jinja2_environment(language: str, xdr_type: str) -> Environment: + """Open a set of templates based on output language""" + match language: + case "C": + environment = Environment( + loader=FileSystemLoader(sys.path[0] + "/templates/C/" + xdr_type + "/"), + trim_blocks=True, + lstrip_blocks=True, + ) + environment.globals["annotate"] = get_xdr_annotate() + environment.globals["public_apis"] = public_apis + environment.globals["pass_by_reference"] = pass_by_reference + return environment + case _: + raise NotImplementedError("Language not supported") + + +def get_jinja2_template( + environment: Environment, template_type: str, template_name: str +) -> Template: + """Retrieve a Jinja2 template for emitting source code""" + return environment.get_template(template_type + "/" + template_name + ".j2") + + +def find_xdr_program_name(root: Specification) -> str: + """Retrieve the RPC program name from an abstract syntax tree""" + raw_name = get_header_name() + if raw_name != "none": + return raw_name.lower() + for definition in root.definitions: + if isinstance(definition.value, _RpcProgram): + raw_name = definition.value.name + return raw_name.lower().removesuffix("_program").removesuffix("_prog") + return "noprog" + + +def header_guard_infix(filename: str) -> str: + """Extract the header guard infix from the specification filename""" + basename = filename.split("/")[-1] + program = basename.replace(".x", "") + return program.upper() + + +def kernel_c_type(spec: _XdrTypeSpecifier) -> str: + """Return name of C type""" + builtin_native_c_type = { + "bool": "bool", + "int": "s32", + "unsigned_int": "u32", + "long": "s32", + "unsigned_long": "u32", + "hyper": "s64", + "unsigned_hyper": "u64", + } + if spec.type_name in builtin_native_c_type: + return builtin_native_c_type[spec.type_name] + return spec.type_name + + +class Boilerplate: + """Base class to generate boilerplate for source files""" + + def __init__(self, language: str, peer: str): + """Initialize an instance of this class""" + raise NotImplementedError("No language support defined") + + def emit_declaration(self, filename: str, root: Specification) -> None: + """Emit declaration header boilerplate""" + raise NotImplementedError("Header boilerplate generation not supported") + + def emit_definition(self, filename: str, root: Specification) -> None: + """Emit definition header boilerplate""" + raise NotImplementedError("Header boilerplate generation not supported") + + def emit_source(self, filename: str, root: Specification) -> None: + """Emit generic source code for this XDR type""" + raise NotImplementedError("Source boilerplate generation not supported") + + +class SourceGenerator: + """Base class to generate header and source code for XDR types""" + + def __init__(self, language: str, peer: str): + """Initialize an instance of this class""" + raise NotImplementedError("No language support defined") + + def emit_declaration(self, node: _XdrAst) -> None: + """Emit one function declaration for this XDR type""" + raise NotImplementedError("Declaration generation not supported") + + def emit_decoder(self, node: _XdrAst) -> None: + """Emit one decoder function for this XDR type""" + raise NotImplementedError("Decoder generation not supported") + + def emit_definition(self, node: _XdrAst) -> None: + """Emit one definition for this XDR type""" + raise NotImplementedError("Definition generation not supported") + + def emit_encoder(self, node: _XdrAst) -> None: + """Emit one encoder function for this XDR type""" + raise NotImplementedError("Encoder generation not supported") diff --git a/tools/net/sunrpc/xdrgen/generators/constant.py b/tools/net/sunrpc/xdrgen/generators/constant.py new file mode 100644 index 000000000000..f2339caf0953 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/generators/constant.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# ex: set filetype=python: + +"""Generate code to handle XDR constants""" + +from generators import SourceGenerator, create_jinja2_environment +from xdr_ast import _XdrConstant + +class XdrConstantGenerator(SourceGenerator): + """Generate source code for XDR constants""" + + def __init__(self, language: str, peer: str): + """Initialize an instance of this class""" + self.environment = create_jinja2_environment(language, "constants") + self.peer = peer + + def emit_definition(self, node: _XdrConstant) -> None: + """Emit one definition for a constant""" + template = self.environment.get_template("definition.j2") + print(template.render(name=node.name, value=node.value)) diff --git a/tools/net/sunrpc/xdrgen/generators/enum.py b/tools/net/sunrpc/xdrgen/generators/enum.py new file mode 100644 index 000000000000..855e43f4ae38 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/generators/enum.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +# ex: set filetype=python: + +"""Generate code to handle XDR enum types""" + +from generators import SourceGenerator, create_jinja2_environment +from xdr_ast import _XdrEnum, public_apis + + +class XdrEnumGenerator(SourceGenerator): + """Generate source code for XDR enum types""" + + def __init__(self, language: str, peer: str): + """Initialize an instance of this class""" + self.environment = create_jinja2_environment(language, "enum") + self.peer = peer + + def emit_declaration(self, node: _XdrEnum) -> None: + """Emit one declaration pair for an XDR enum type""" + if node.name in public_apis: + template = self.environment.get_template("declaration/close.j2") + print(template.render(name=node.name)) + + def emit_definition(self, node: _XdrEnum) -> None: + """Emit one definition for an XDR enum type""" + template = self.environment.get_template("definition/open.j2") + print(template.render(name=node.name)) + + template = self.environment.get_template("definition/enumerator.j2") + for enumerator in node.enumerators: + print(template.render(name=enumerator.name, value=enumerator.value)) + + template = self.environment.get_template("definition/close.j2") + print(template.render(name=node.name)) + + def emit_decoder(self, node: _XdrEnum) -> None: + """Emit one decoder function for an XDR enum type""" + template = self.environment.get_template("decoder/enum.j2") + print(template.render(name=node.name)) + + def emit_encoder(self, node: _XdrEnum) -> None: + """Emit one encoder function for an XDR enum type""" + template = self.environment.get_template("encoder/enum.j2") + print(template.render(name=node.name)) diff --git a/tools/net/sunrpc/xdrgen/generators/header_bottom.py b/tools/net/sunrpc/xdrgen/generators/header_bottom.py new file mode 100644 index 000000000000..4b55b282dfc0 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/generators/header_bottom.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# ex: set filetype=python: + +"""Generate header bottom boilerplate""" + +import os.path +import time + +from generators import Boilerplate, header_guard_infix +from generators import create_jinja2_environment, get_jinja2_template +from xdr_ast import Specification + + +class XdrHeaderBottomGenerator(Boilerplate): + """Generate header boilerplate""" + + def __init__(self, language: str, peer: str): + """Initialize an instance of this class""" + self.environment = create_jinja2_environment(language, "header_bottom") + self.peer = peer + + def emit_declaration(self, filename: str, root: Specification) -> None: + """Emit the bottom header guard""" + template = get_jinja2_template(self.environment, "declaration", "header") + print(template.render(infix=header_guard_infix(filename))) + + def emit_definition(self, filename: str, root: Specification) -> None: + """Emit the bottom header guard""" + template = get_jinja2_template(self.environment, "definition", "header") + print(template.render(infix=header_guard_infix(filename))) + + def emit_source(self, filename: str, root: Specification) -> None: + pass diff --git a/tools/net/sunrpc/xdrgen/generators/header_top.py b/tools/net/sunrpc/xdrgen/generators/header_top.py new file mode 100644 index 000000000000..c6bc21c71f19 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/generators/header_top.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +# ex: set filetype=python: + +"""Generate header top boilerplate""" + +import os.path +import time + +from generators import Boilerplate, header_guard_infix +from generators import create_jinja2_environment, get_jinja2_template +from xdr_ast import Specification + + +class XdrHeaderTopGenerator(Boilerplate): + """Generate header boilerplate""" + + def __init__(self, language: str, peer: str): + """Initialize an instance of this class""" + self.environment = create_jinja2_environment(language, "header_top") + self.peer = peer + + def emit_declaration(self, filename: str, root: Specification) -> None: + """Emit the top header guard""" + template = get_jinja2_template(self.environment, "declaration", "header") + print( + template.render( + infix=header_guard_infix(filename), + filename=filename, + mtime=time.ctime(os.path.getmtime(filename)), + ) + ) + + def emit_definition(self, filename: str, root: Specification) -> None: + """Emit the top header guard""" + template = get_jinja2_template(self.environment, "definition", "header") + print( + template.render( + infix=header_guard_infix(filename), + filename=filename, + mtime=time.ctime(os.path.getmtime(filename)), + ) + ) + + def emit_source(self, filename: str, root: Specification) -> None: + pass diff --git a/tools/net/sunrpc/xdrgen/generators/pointer.py b/tools/net/sunrpc/xdrgen/generators/pointer.py new file mode 100644 index 000000000000..b0b27f1819c8 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/generators/pointer.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 +# ex: set filetype=python: + +"""Generate code to handle XDR pointer types""" + +from jinja2 import Environment + +from generators import SourceGenerator, kernel_c_type +from generators import create_jinja2_environment, get_jinja2_template + +from xdr_ast import _XdrBasic, _XdrVariableLengthString +from xdr_ast import _XdrFixedLengthOpaque, _XdrVariableLengthOpaque +from xdr_ast import _XdrFixedLengthArray, _XdrVariableLengthArray +from xdr_ast import _XdrOptionalData, _XdrPointer, _XdrDeclaration +from xdr_ast import public_apis + + +def emit_pointer_declaration(environment: Environment, node: _XdrPointer) -> None: + """Emit a declaration pair for an XDR pointer type""" + if node.name in public_apis: + template = get_jinja2_template(environment, "declaration", "close") + print(template.render(name=node.name)) + + +def emit_pointer_member_definition( + environment: Environment, field: _XdrDeclaration +) -> None: + """Emit a definition for one field in an XDR struct""" + if isinstance(field, _XdrBasic): + template = get_jinja2_template(environment, "definition", field.template) + print( + template.render( + name=field.name, + type=kernel_c_type(field.spec), + classifier=field.spec.c_classifier, + ) + ) + elif isinstance(field, _XdrFixedLengthOpaque): + template = get_jinja2_template(environment, "definition", field.template) + print( + template.render( + name=field.name, + size=field.size, + ) + ) + elif isinstance(field, _XdrVariableLengthOpaque): + template = get_jinja2_template(environment, "definition", field.template) + print(template.render(name=field.name)) + elif isinstance(field, _XdrVariableLengthString): + template = get_jinja2_template(environment, "definition", field.template) + print(template.render(name=field.name)) + elif isinstance(field, _XdrFixedLengthArray): + template = get_jinja2_template(environment, "definition", field.template) + print( + template.render( + name=field.name, + type=kernel_c_type(field.spec), + size=field.size, + ) + ) + elif isinstance(field, _XdrVariableLengthArray): + template = get_jinja2_template(environment, "definition", field.template) + print( + template.render( + name=field.name, + type=kernel_c_type(field.spec), + classifier=field.spec.c_classifier, + ) + ) + elif isinstance(field, _XdrOptionalData): + template = get_jinja2_template(environment, "definition", field.template) + print( + template.render( + name=field.name, + type=kernel_c_type(field.spec), + classifier=field.spec.c_classifier, + ) + ) + + +def emit_pointer_definition(environment: Environment, node: _XdrPointer) -> None: + """Emit a definition for an XDR pointer type""" + template = get_jinja2_template(environment, "definition", "open") + print(template.render(name=node.name)) + + for field in node.fields[0:-1]: + emit_pointer_member_definition(environment, field) + + template = get_jinja2_template(environment, "definition", "close") + print(template.render(name=node.name)) + + +def emit_pointer_member_decoder( + environment: Environment, field: _XdrDeclaration +) -> None: + """Emit a decoder for one field in an XDR pointer""" + if isinstance(field, _XdrBasic): + template = get_jinja2_template(environment, "decoder", field.template) + print( + template.render( + name=field.name, + type=field.spec.type_name, + classifier=field.spec.c_classifier, + ) + ) + elif isinstance(field, _XdrFixedLengthOpaque): + template = get_jinja2_template(environment, "decoder", field.template) + print( + template.render( + name=field.name, + size=field.size, + ) + ) + elif isinstance(field, _XdrVariableLengthOpaque): + template = get_jinja2_template(environment, "decoder", field.template) + print( + template.render( + name=field.name, + maxsize=field.maxsize, + ) + ) + elif isinstance(field, _XdrVariableLengthString): + template = get_jinja2_template(environment, "decoder", field.template) + print( + template.render( + name=field.name, + maxsize=field.maxsize, + ) + ) + elif isinstance(field, _XdrFixedLengthArray): + template = get_jinja2_template(environment, "decoder", field.template) + print( + template.render( + name=field.name, + type=field.spec.type_name, + size=field.size, + classifier=field.spec.c_classifier, + ) + ) + elif isinstance(field, _XdrVariableLengthArray): + template = get_jinja2_template(environment, "decoder", field.template) + print( + template.render( + name=field.name, + type=field.spec.type_name, + maxsize=field.maxsize, + classifier=field.spec.c_classifier, + ) + ) + elif isinstance(field, _XdrOptionalData): + template = get_jinja2_template(environment, "decoder", field.template) + print( + template.render( + name=field.name, + type=field.spec.type_name, + classifier=field.spec.c_classifier, + ) + ) + + +def emit_pointer_decoder(environment: Environment, node: _XdrPointer) -> None: + """Emit one decoder function for an XDR pointer type""" + template = get_jinja2_template(environment, "decoder", "open") + print(template.render(name=node.name)) + + for field in node.fields[0:-1]: + emit_pointer_member_decoder(environment, field) + + template = get_jinja2_template(environment, "decoder", "close") + print(template.render()) + + +def emit_pointer_member_encoder( + environment: Environment, field: _XdrDeclaration +) -> None: + """Emit an encoder for one field in a XDR pointer""" + if isinstance(field, _XdrBasic): + template = get_jinja2_template(environment, "encoder", field.template) + print( + template.render( + name=field.name, + type=field.spec.type_name, + ) + ) + elif isinstance(field, _XdrFixedLengthOpaque): + template = get_jinja2_template(environment, "encoder", field.template) + print( + template.render( + name=field.name, + size=field.size, + ) + ) + elif isinstance(field, _XdrVariableLengthOpaque): + template = get_jinja2_template(environment, "encoder", field.template) + print( + template.render( + name=field.name, + maxsize=field.maxsize, + ) + ) + elif isinstance(field, _XdrVariableLengthString): + template = get_jinja2_template(environment, "encoder", field.template) + print( + template.render( + name=field.name, + maxsize=field.maxsize, + ) + ) + elif isinstance(field, _XdrFixedLengthArray): + template = get_jinja2_template(environment, "encoder", field.template) + print( + template.render( + name=field.name, + type=field.spec.type_name, + size=field.size, + ) + ) + elif isinstance(field, _XdrVariableLengthArray): + template = get_jinja2_template(environment, "encoder", field.template) + print( + template.render( + name=field.name, + type=field.spec.type_name, + maxsize=field.maxsize, + ) + ) + elif isinstance(field, _XdrOptionalData): + template = get_jinja2_template(environment, "encoder", field.template) + print( + template.render( + name=field.name, + type=field.spec.type_name, + classifier=field.spec.c_classifier, + ) + ) + + +def emit_pointer_encoder(environment: Environment, node: _XdrPointer) -> None: + """Emit one encoder function for an XDR pointer type""" + template = get_jinja2_template(environment, "encoder", "open") + print(template.render(name=node.name)) + + for field in node.fields[0:-1]: + emit_pointer_member_encoder(environment, field) + + template = get_jinja2_template(environment, "encoder", "close") + print(template.render()) + + +class XdrPointerGenerator(SourceGenerator): + """Generate source code for XDR pointer""" + + def __init__(self, language: str, peer: str): + """Initialize an instance of this class""" + self.environment = create_jinja2_environment(language, "pointer") + self.peer = peer + + def emit_declaration(self, node: _XdrPointer) -> None: + """Emit one declaration pair for an XDR pointer type""" + emit_pointer_declaration(self.environment, node) + + def emit_definition(self, node: _XdrPointer) -> None: + """Emit one declaration for an XDR pointer type""" + emit_pointer_definition(self.environment, node) + + def emit_decoder(self, node: _XdrPointer) -> None: + """Emit one decoder function for an XDR pointer type""" + emit_pointer_decoder(self.environment, node) + + def emit_encoder(self, node: _XdrPointer) -> None: + """Emit one encoder function for an XDR pointer type""" + emit_pointer_encoder(self.environment, node) diff --git a/tools/net/sunrpc/xdrgen/generators/program.py b/tools/net/sunrpc/xdrgen/generators/program.py new file mode 100644 index 000000000000..83b0ecbae86f --- /dev/null +++ b/tools/net/sunrpc/xdrgen/generators/program.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +# ex: set filetype=python: + +"""Generate code for an RPC program's procedures""" + +from jinja2 import Environment + +from generators import SourceGenerator, create_jinja2_environment +from xdr_ast import _RpcProgram, _RpcVersion, excluded_apis + + +def emit_version_definitions( + environment: Environment, program: str, version: _RpcVersion +) -> None: + """Emit procedure numbers for each RPC version's procedures""" + template = environment.get_template("definition/open.j2") + print(template.render(program=program.upper())) + + template = environment.get_template("definition/procedure.j2") + for procedure in version.procedures: + if procedure.name not in excluded_apis: + print( + template.render( + name=procedure.name, + value=procedure.number, + ) + ) + + template = environment.get_template("definition/close.j2") + print(template.render()) + + +def emit_version_declarations( + environment: Environment, program: str, version: _RpcVersion +) -> None: + """Emit declarations for each RPC version's procedures""" + arguments = set() + for procedure in version.procedures: + if procedure.name not in excluded_apis: + arguments.add(procedure.argument.type_name) + if len(arguments) > 0: + print("") + template = environment.get_template("declaration/argument.j2") + for argument in arguments: + print(template.render(program=program, argument=argument)) + + results = set() + for procedure in version.procedures: + if procedure.name not in excluded_apis: + results.add(procedure.result.type_name) + if len(results) > 0: + print("") + template = environment.get_template("declaration/result.j2") + for result in results: + print(template.render(program=program, result=result)) + + +def emit_version_argument_decoders( + environment: Environment, program: str, version: _RpcVersion +) -> None: + """Emit server argument decoders for each RPC version's procedures""" + arguments = set() + for procedure in version.procedures: + if procedure.name not in excluded_apis: + arguments.add(procedure.argument.type_name) + + template = environment.get_template("decoder/argument.j2") + for argument in arguments: + print(template.render(program=program, argument=argument)) + + +def emit_version_result_decoders( + environment: Environment, program: str, version: _RpcVersion +) -> None: + """Emit client result decoders for each RPC version's procedures""" + results = set() + for procedure in version.procedures: + if procedure.name not in excluded_apis: + results.add(procedure.result.type_name) + + template = environment.get_template("decoder/result.j2") + for result in results: + print(template.render(program=program, result=result)) + + +def emit_version_argument_encoders( + environment: Environment, program: str, version: _RpcVersion +) -> None: + """Emit client argument encoders for each RPC version's procedures""" + arguments = set() + for procedure in version.procedures: + if procedure.name not in excluded_apis: + arguments.add(procedure.argument.type_name) + + template = environment.get_template("encoder/argument.j2") + for argument in arguments: + print(template.render(program=program, argument=argument)) + + +def emit_version_result_encoders( + environment: Environment, program: str, version: _RpcVersion +) -> None: + """Emit server result encoders for each RPC version's procedures""" + results = set() + for procedure in version.procedures: + if procedure.name not in excluded_apis: + results.add(procedure.result.type_name) + + template = environment.get_template("encoder/result.j2") + for result in results: + print(template.render(program=program, result=result)) + + +class XdrProgramGenerator(SourceGenerator): + """Generate source code for an RPC program's procedures""" + + def __init__(self, language: str, peer: str): + """Initialize an instance of this class""" + self.environment = create_jinja2_environment(language, "program") + self.peer = peer + + def emit_definition(self, node: _RpcProgram) -> None: + """Emit procedure numbers for each of an RPC programs's procedures""" + raw_name = node.name + program = raw_name.lower().removesuffix("_program").removesuffix("_prog") + + for version in node.versions: + emit_version_definitions(self.environment, program, version) + + def emit_declaration(self, node: _RpcProgram) -> None: + """Emit a declaration pair for each of an RPC programs's procedures""" + raw_name = node.name + program = raw_name.lower().removesuffix("_program").removesuffix("_prog") + + for version in node.versions: + emit_version_declarations(self.environment, program, version) + + def emit_decoder(self, node: _RpcProgram) -> None: + """Emit all decoder functions for an RPC program's procedures""" + raw_name = node.name + program = raw_name.lower().removesuffix("_program").removesuffix("_prog") + match self.peer: + case "server": + for version in node.versions: + emit_version_argument_decoders( + self.environment, program, version, + ) + case "client": + for version in node.versions: + emit_version_result_decoders( + self.environment, program, version, + ) + + def emit_encoder(self, node: _RpcProgram) -> None: + """Emit all encoder functions for an RPC program's procedures""" + raw_name = node.name + program = raw_name.lower().removesuffix("_program").removesuffix("_prog") + match self.peer: + case "server": + for version in node.versions: + emit_version_result_encoders( + self.environment, program, version, + ) + case "client": + for version in node.versions: + emit_version_argument_encoders( + self.environment, program, version, + ) diff --git a/tools/net/sunrpc/xdrgen/generators/source_top.py b/tools/net/sunrpc/xdrgen/generators/source_top.py new file mode 100644 index 000000000000..bcf47d93d6f1 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/generators/source_top.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +# ex: set filetype=python: + +"""Generate source code boilerplate""" + +import os.path +import time + +from generators import Boilerplate +from generators import find_xdr_program_name, create_jinja2_environment +from xdr_ast import _RpcProgram, Specification, get_header_name + + +class XdrSourceTopGenerator(Boilerplate): + """Generate source code boilerplate""" + + def __init__(self, language: str, peer: str): + """Initialize an instance of this class""" + self.environment = create_jinja2_environment(language, "source_top") + self.peer = peer + + def emit_source(self, filename: str, root: Specification) -> None: + """Emit the top source boilerplate""" + name = find_xdr_program_name(root) + template = self.environment.get_template(self.peer + ".j2") + print( + template.render( + program=name, + filename=filename, + mtime=time.ctime(os.path.getmtime(filename)), + ) + ) diff --git a/tools/net/sunrpc/xdrgen/generators/struct.py b/tools/net/sunrpc/xdrgen/generators/struct.py new file mode 100644 index 000000000000..b694cd470829 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/generators/struct.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 +# ex: set filetype=python: + +"""Generate code to handle XDR struct types""" + +from jinja2 import Environment + +from generators import SourceGenerator, kernel_c_type +from generators import create_jinja2_environment, get_jinja2_template + +from xdr_ast import _XdrBasic, _XdrVariableLengthString +from xdr_ast import _XdrFixedLengthOpaque, _XdrVariableLengthOpaque +from xdr_ast import _XdrFixedLengthArray, _XdrVariableLengthArray +from xdr_ast import _XdrOptionalData, _XdrStruct, _XdrDeclaration +from xdr_ast import public_apis + + +def emit_struct_declaration(environment: Environment, node: _XdrStruct) -> None: + """Emit one declaration pair for an XDR struct type""" + if node.name in public_apis: + template = get_jinja2_template(environment, "declaration", "close") + print(template.render(name=node.name)) + + +def emit_struct_member_definition( + environment: Environment, field: _XdrDeclaration +) -> None: + """Emit a definition for one field in an XDR struct""" + if isinstance(field, _XdrBasic): + template = get_jinja2_template(environment, "definition", field.template) + print( + template.render( + name=field.name, + type=kernel_c_type(field.spec), + classifier=field.spec.c_classifier, + ) + ) + elif isinstance(field, _XdrFixedLengthOpaque): + template = get_jinja2_template(environment, "definition", field.template) + print( + template.render( + name=field.name, + size=field.size, + ) + ) + elif isinstance(field, _XdrVariableLengthOpaque): + template = get_jinja2_template(environment, "definition", field.template) + print(template.render(name=field.name)) + elif isinstance(field, _XdrVariableLengthString): + template = get_jinja2_template(environment, "definition", field.template) + print(template.render(name=field.name)) + elif isinstance(field, _XdrFixedLengthArray): + template = get_jinja2_template(environment, "definition", field.template) + print( + template.render( + name=field.name, + type=kernel_c_type(field.spec), + size=field.size, + ) + ) + elif isinstance(field, _XdrVariableLengthArray): + template = get_jinja2_template(environment, "definition", field.template) + print( + template.render( + name=field.name, + type=kernel_c_type(field.spec), + classifier=field.spec.c_classifier, + ) + ) + elif isinstance(field, _XdrOptionalData): + template = get_jinja2_template(environment, "definition", field.template) + print( + template.render( + name=field.name, + type=kernel_c_type(field.spec), + classifier=field.spec.c_classifier, + ) + ) + + +def emit_struct_definition(environment: Environment, node: _XdrStruct) -> None: + """Emit one definition for an XDR struct type""" + template = get_jinja2_template(environment, "definition", "open") + print(template.render(name=node.name)) + + for field in node.fields: + emit_struct_member_definition(environment, field) + + template = get_jinja2_template(environment, "definition", "close") + print(template.render(name=node.name)) + + +def emit_struct_member_decoder( + environment: Environment, field: _XdrDeclaration +) -> None: + """Emit a decoder for one field in an XDR struct""" + if isinstance(field, _XdrBasic): + template = get_jinja2_template(environment, "decoder", field.template) + print( + template.render( + name=field.name, + type=field.spec.type_name, + classifier=field.spec.c_classifier, + ) + ) + elif isinstance(field, _XdrFixedLengthOpaque): + template = get_jinja2_template(environment, "decoder", field.template) + print( + template.render( + name=field.name, + size=field.size, + ) + ) + elif isinstance(field, _XdrVariableLengthOpaque): + template = get_jinja2_template(environment, "decoder", field.template) + print( + template.render( + name=field.name, + maxsize=field.maxsize, + ) + ) + elif isinstance(field, _XdrVariableLengthString): + template = get_jinja2_template(environment, "decoder", field.template) + print( + template.render( + name=field.name, + maxsize=field.maxsize, + ) + ) + elif isinstance(field, _XdrFixedLengthArray): + template = get_jinja2_template(environment, "decoder", field.template) + print( + template.render( + name=field.name, + type=field.spec.type_name, + size=field.size, + classifier=field.spec.c_classifier, + ) + ) + elif isinstance(field, _XdrVariableLengthArray): + template = get_jinja2_template(environment, "decoder", field.template) + print( + template.render( + name=field.name, + type=field.spec.type_name, + maxsize=field.maxsize, + classifier=field.spec.c_classifier, + ) + ) + elif isinstance(field, _XdrOptionalData): + template = get_jinja2_template(environment, "decoder", field.template) + print( + template.render( + name=field.name, + type=field.spec.type_name, + classifier=field.spec.c_classifier, + ) + ) + + +def emit_struct_decoder(environment: Environment, node: _XdrStruct) -> None: + """Emit one decoder function for an XDR struct type""" + template = get_jinja2_template(environment, "decoder", "open") + print(template.render(name=node.name)) + + for field in node.fields: + emit_struct_member_decoder(environment, field) + + template = get_jinja2_template(environment, "decoder", "close") + print(template.render()) + + +def emit_struct_member_encoder( + environment: Environment, field: _XdrDeclaration +) -> None: + """Emit an encoder for one field in an XDR struct""" + if isinstance(field, _XdrBasic): + template = get_jinja2_template(environment, "encoder", field.template) + print( + template.render( + name=field.name, + type=field.spec.type_name, + ) + ) + elif isinstance(field, _XdrFixedLengthOpaque): + template = get_jinja2_template(environment, "encoder", field.template) + print( + template.render( + name=field.name, + size=field.size, + ) + ) + elif isinstance(field, _XdrVariableLengthOpaque): + template = get_jinja2_template(environment, "encoder", field.template) + print( + template.render( + name=field.name, + maxsize=field.maxsize, + ) + ) + elif isinstance(field, _XdrVariableLengthString): + template = get_jinja2_template(environment, "encoder", field.template) + print( + template.render( + name=field.name, + maxsize=field.maxsize, + ) + ) + elif isinstance(field, _XdrFixedLengthArray): + template = get_jinja2_template(environment, "encoder", field.template) + print( + template.render( + name=field.name, + type=field.spec.type_name, + size=field.size, + ) + ) + elif isinstance(field, _XdrVariableLengthArray): + template = get_jinja2_template(environment, "encoder", field.template) + print( + template.render( + name=field.name, + type=field.spec.type_name, + maxsize=field.maxsize, + ) + ) + elif isinstance(field, _XdrOptionalData): + template = get_jinja2_template(environment, "encoder", field.template) + print( + template.render( + name=field.name, + type=field.spec.type_name, + classifier=field.spec.c_classifier, + ) + ) + + +def emit_struct_encoder(environment: Environment, node: _XdrStruct) -> None: + """Emit one encoder function for an XDR struct type""" + template = get_jinja2_template(environment, "encoder", "open") + print(template.render(name=node.name)) + + for field in node.fields: + emit_struct_member_encoder(environment, field) + + template = get_jinja2_template(environment, "encoder", "close") + print(template.render()) + + +class XdrStructGenerator(SourceGenerator): + """Generate source code for XDR structs""" + + def __init__(self, language: str, peer: str): + """Initialize an instance of this class""" + self.environment = create_jinja2_environment(language, "struct") + self.peer = peer + + def emit_declaration(self, node: _XdrStruct) -> None: + """Emit one declaration pair for an XDR struct type""" + emit_struct_declaration(self.environment, node) + + def emit_definition(self, node: _XdrStruct) -> None: + """Emit one definition for an XDR struct type""" + emit_struct_definition(self.environment, node) + + def emit_decoder(self, node: _XdrStruct) -> None: + """Emit one decoder function for an XDR struct type""" + emit_struct_decoder(self.environment, node) + + def emit_encoder(self, node: _XdrStruct) -> None: + """Emit one encoder function for an XDR struct type""" + emit_struct_encoder(self.environment, node) diff --git a/tools/net/sunrpc/xdrgen/generators/typedef.py b/tools/net/sunrpc/xdrgen/generators/typedef.py new file mode 100644 index 000000000000..85a1b2303333 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/generators/typedef.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python3 +# ex: set filetype=python: + +"""Generate code to handle XDR typedefs""" + +from jinja2 import Environment + +from generators import SourceGenerator, kernel_c_type +from generators import create_jinja2_environment, get_jinja2_template + +from xdr_ast import _XdrBasic, _XdrTypedef, _XdrVariableLengthString +from xdr_ast import _XdrFixedLengthOpaque, _XdrVariableLengthOpaque +from xdr_ast import _XdrFixedLengthArray, _XdrVariableLengthArray +from xdr_ast import _XdrOptionalData, _XdrVoid, _XdrDeclaration +from xdr_ast import public_apis + + +def emit_typedef_declaration(environment: Environment, node: _XdrDeclaration) -> None: + """Emit a declaration pair for one XDR typedef""" + if node.name not in public_apis: + return + if isinstance(node, _XdrBasic): + template = get_jinja2_template(environment, "declaration", node.template) + print( + template.render( + name=node.name, + type=kernel_c_type(node.spec), + classifier=node.spec.c_classifier, + ) + ) + elif isinstance(node, _XdrVariableLengthString): + template = get_jinja2_template(environment, "declaration", node.template) + print(template.render(name=node.name)) + elif isinstance(node, _XdrFixedLengthOpaque): + template = get_jinja2_template(environment, "declaration", node.template) + print(template.render(name=node.name, size=node.size)) + elif isinstance(node, _XdrVariableLengthOpaque): + template = get_jinja2_template(environment, "declaration", node.template) + print(template.render(name=node.name)) + elif isinstance(node, _XdrFixedLengthArray): + template = get_jinja2_template(environment, "declaration", node.template) + print( + template.render( + name=node.name, + type=node.spec.type_name, + size=node.size, + ) + ) + elif isinstance(node, _XdrVariableLengthArray): + template = get_jinja2_template(environment, "declaration", node.template) + print( + template.render( + name=node.name, + type=node.spec.type_name, + classifier=node.spec.c_classifier, + ) + ) + elif isinstance(node, _XdrOptionalData): + raise NotImplementedError(" typedef not yet implemented") + elif isinstance(node, _XdrVoid): + raise NotImplementedError(" typedef not yet implemented") + else: + raise NotImplementedError("typedef: type not recognized") + + +def emit_type_definition(environment: Environment, node: _XdrDeclaration) -> None: + """Emit a definition for one XDR typedef""" + if isinstance(node, _XdrBasic): + template = get_jinja2_template(environment, "definition", node.template) + print( + template.render( + name=node.name, + type=kernel_c_type(node.spec), + classifier=node.spec.c_classifier, + ) + ) + elif isinstance(node, _XdrVariableLengthString): + template = get_jinja2_template(environment, "definition", node.template) + print(template.render(name=node.name)) + elif isinstance(node, _XdrFixedLengthOpaque): + template = get_jinja2_template(environment, "definition", node.template) + print(template.render(name=node.name, size=node.size)) + elif isinstance(node, _XdrVariableLengthOpaque): + template = get_jinja2_template(environment, "definition", node.template) + print(template.render(name=node.name)) + elif isinstance(node, _XdrFixedLengthArray): + template = get_jinja2_template(environment, "definition", node.template) + print( + template.render( + name=node.name, + type=node.spec.type_name, + size=node.size, + ) + ) + elif isinstance(node, _XdrVariableLengthArray): + template = get_jinja2_template(environment, "definition", node.template) + print( + template.render( + name=node.name, + type=node.spec.type_name, + classifier=node.spec.c_classifier, + ) + ) + elif isinstance(node, _XdrOptionalData): + raise NotImplementedError(" typedef not yet implemented") + elif isinstance(node, _XdrVoid): + raise NotImplementedError(" typedef not yet implemented") + else: + raise NotImplementedError("typedef: type not recognized") + + +def emit_typedef_decoder(environment: Environment, node: _XdrDeclaration) -> None: + """Emit a decoder function for one XDR typedef""" + if isinstance(node, _XdrBasic): + template = get_jinja2_template(environment, "decoder", node.template) + print( + template.render( + name=node.name, + type=node.spec.type_name, + ) + ) + elif isinstance(node, _XdrVariableLengthString): + template = get_jinja2_template(environment, "decoder", node.template) + print( + template.render( + name=node.name, + maxsize=node.maxsize, + ) + ) + elif isinstance(node, _XdrFixedLengthOpaque): + template = get_jinja2_template(environment, "decoder", node.template) + print( + template.render( + name=node.name, + size=node.size, + ) + ) + elif isinstance(node, _XdrVariableLengthOpaque): + template = get_jinja2_template(environment, "decoder", node.template) + print( + template.render( + name=node.name, + maxsize=node.maxsize, + ) + ) + elif isinstance(node, _XdrFixedLengthArray): + template = get_jinja2_template(environment, "decoder", node.template) + print( + template.render( + name=node.name, + type=node.spec.type_name, + size=node.size, + classifier=node.spec.c_classifier, + ) + ) + elif isinstance(node, _XdrVariableLengthArray): + template = get_jinja2_template(environment, "decoder", node.template) + print( + template.render( + name=node.name, + type=node.spec.type_name, + maxsize=node.maxsize, + ) + ) + elif isinstance(node, _XdrOptionalData): + raise NotImplementedError(" typedef not yet implemented") + elif isinstance(node, _XdrVoid): + raise NotImplementedError(" typedef not yet implemented") + else: + raise NotImplementedError("typedef: type not recognized") + + +def emit_typedef_encoder(environment: Environment, node: _XdrDeclaration) -> None: + """Emit an encoder function for one XDR typedef""" + if isinstance(node, _XdrBasic): + template = get_jinja2_template(environment, "encoder", node.template) + print( + template.render( + name=node.name, + type=node.spec.type_name, + ) + ) + elif isinstance(node, _XdrVariableLengthString): + template = get_jinja2_template(environment, "encoder", node.template) + print( + template.render( + name=node.name, + maxsize=node.maxsize, + ) + ) + elif isinstance(node, _XdrFixedLengthOpaque): + template = get_jinja2_template(environment, "encoder", node.template) + print( + template.render( + name=node.name, + size=node.size, + ) + ) + elif isinstance(node, _XdrVariableLengthOpaque): + template = get_jinja2_template(environment, "encoder", node.template) + print( + template.render( + name=node.name, + maxsize=node.maxsize, + ) + ) + elif isinstance(node, _XdrFixedLengthArray): + template = get_jinja2_template(environment, "encoder", node.template) + print( + template.render( + name=node.name, + type=node.spec.type_name, + size=node.size, + ) + ) + elif isinstance(node, _XdrVariableLengthArray): + template = get_jinja2_template(environment, "encoder", node.template) + print( + template.render( + name=node.name, + type=node.spec.type_name, + maxsize=node.maxsize, + ) + ) + elif isinstance(node, _XdrOptionalData): + raise NotImplementedError(" typedef not yet implemented") + elif isinstance(node, _XdrVoid): + raise NotImplementedError(" typedef not yet implemented") + else: + raise NotImplementedError("typedef: type not recognized") + + +class XdrTypedefGenerator(SourceGenerator): + """Generate source code for XDR typedefs""" + + def __init__(self, language: str, peer: str): + """Initialize an instance of this class""" + self.environment = create_jinja2_environment(language, "typedef") + self.peer = peer + + def emit_declaration(self, node: _XdrTypedef) -> None: + """Emit one declaration pair for an XDR enum type""" + emit_typedef_declaration(self.environment, node.declaration) + + def emit_definition(self, node: _XdrTypedef) -> None: + """Emit one definition for an XDR typedef""" + emit_type_definition(self.environment, node.declaration) + + def emit_decoder(self, node: _XdrTypedef) -> None: + """Emit one decoder function for an XDR typedef""" + emit_typedef_decoder(self.environment, node.declaration) + + def emit_encoder(self, node: _XdrTypedef) -> None: + """Emit one encoder function for an XDR typedef""" + emit_typedef_encoder(self.environment, node.declaration) diff --git a/tools/net/sunrpc/xdrgen/generators/union.py b/tools/net/sunrpc/xdrgen/generators/union.py new file mode 100644 index 000000000000..7974967bbb9f --- /dev/null +++ b/tools/net/sunrpc/xdrgen/generators/union.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +# ex: set filetype=python: + +"""Generate code to handle XDR unions""" + +from jinja2 import Environment + +from generators import SourceGenerator +from generators import create_jinja2_environment, get_jinja2_template + +from xdr_ast import _XdrBasic, _XdrUnion, _XdrVoid +from xdr_ast import _XdrDeclaration, _XdrCaseSpec, public_apis + + +def emit_union_declaration(environment: Environment, node: _XdrUnion) -> None: + """Emit one declaration pair for an XDR union type""" + if node.name in public_apis: + template = get_jinja2_template(environment, "declaration", "close") + print(template.render(name=node.name)) + + +def emit_union_switch_spec_definition( + environment: Environment, node: _XdrDeclaration +) -> None: + """Emit a definition for an XDR union's discriminant""" + assert isinstance(node, _XdrBasic) + template = get_jinja2_template(environment, "definition", "switch_spec") + print( + template.render( + name=node.name, + type=node.spec.type_name, + classifier=node.spec.c_classifier, + ) + ) + + +def emit_union_case_spec_definition( + environment: Environment, node: _XdrDeclaration +) -> None: + """Emit a definition for an XDR union's case arm""" + if isinstance(node.arm, _XdrVoid): + return + assert isinstance(node.arm, _XdrBasic) + template = get_jinja2_template(environment, "definition", "case_spec") + print( + template.render( + name=node.arm.name, + type=node.arm.spec.type_name, + classifier=node.arm.spec.c_classifier, + ) + ) + + +def emit_union_definition(environment: Environment, node: _XdrUnion) -> None: + """Emit one XDR union definition""" + template = get_jinja2_template(environment, "definition", "open") + print(template.render(name=node.name)) + + emit_union_switch_spec_definition(environment, node.discriminant) + + for case in node.cases: + emit_union_case_spec_definition(environment, case) + + if node.default is not None: + emit_union_case_spec_definition(environment, node.default) + + template = get_jinja2_template(environment, "definition", "close") + print(template.render(name=node.name)) + + +def emit_union_switch_spec_decoder( + environment: Environment, node: _XdrDeclaration +) -> None: + """Emit a decoder for an XDR union's discriminant""" + assert isinstance(node, _XdrBasic) + template = get_jinja2_template(environment, "decoder", "switch_spec") + print(template.render(name=node.name, type=node.spec.type_name)) + + +def emit_union_case_spec_decoder(environment: Environment, node: _XdrCaseSpec) -> None: + """Emit decoder functions for an XDR union's case arm""" + + if isinstance(node.arm, _XdrVoid): + return + + template = get_jinja2_template(environment, "decoder", "case_spec") + for case in node.values: + print(template.render(case=case)) + + assert isinstance(node.arm, _XdrBasic) + template = get_jinja2_template(environment, "decoder", node.arm.template) + print( + template.render( + name=node.arm.name, + type=node.arm.spec.type_name, + classifier=node.arm.spec.c_classifier, + ) + ) + + template = get_jinja2_template(environment, "decoder", "break") + print(template.render()) + + +def emit_union_default_spec_decoder(environment: Environment, node: _XdrUnion) -> None: + """Emit a decoder function for an XDR union's default arm""" + default_case = node.default + + # Avoid a gcc warning about a default case with boolean discriminant + if default_case is None and node.discriminant.spec.type_name == "bool": + return + + template = get_jinja2_template(environment, "decoder", "default_spec") + print(template.render()) + + if default_case is None or isinstance(default_case.arm, _XdrVoid): + template = get_jinja2_template(environment, "decoder", "break") + print(template.render()) + return + + assert isinstance(default_case.arm, _XdrBasic) + template = get_jinja2_template(environment, "decoder", default_case.arm.template) + print( + template.render( + name=default_case.arm.name, + type=default_case.arm.spec.type_name, + classifier=default_case.arm.spec.c_classifier, + ) + ) + + +def emit_union_decoder(environment: Environment, node: _XdrUnion) -> None: + """Emit one XDR union decoder""" + template = get_jinja2_template(environment, "decoder", "open") + print(template.render(name=node.name)) + + emit_union_switch_spec_decoder(environment, node.discriminant) + + for case in node.cases: + emit_union_case_spec_decoder(environment, case) + + emit_union_default_spec_decoder(environment, node) + + template = get_jinja2_template(environment, "decoder", "close") + print(template.render()) + + +def emit_union_switch_spec_encoder( + environment: Environment, node: _XdrDeclaration +) -> None: + """Emit an encoder for an XDR union's discriminant""" + assert isinstance(node, _XdrBasic) + template = get_jinja2_template(environment, "encoder", "switch_spec") + print(template.render(name=node.name, type=node.spec.type_name)) + + +def emit_union_case_spec_encoder(environment: Environment, node: _XdrCaseSpec) -> None: + """Emit encoder functions for an XDR union's case arm""" + + if isinstance(node.arm, _XdrVoid): + return + + template = get_jinja2_template(environment, "encoder", "case_spec") + for case in node.values: + print(template.render(case=case)) + + assert isinstance(node.arm, _XdrBasic) + template = get_jinja2_template(environment, "encoder", node.arm.template) + print( + template.render( + name=node.arm.name, + type=node.arm.spec.type_name, + ) + ) + + template = get_jinja2_template(environment, "encoder", "break") + print(template.render()) + + +def emit_union_default_spec_encoder(environment: Environment, node: _XdrUnion) -> None: + """Emit an encoder function for an XDR union's default arm""" + default_case = node.default + + # Avoid a gcc warning about a default case with boolean discriminant + if default_case is None and node.discriminant.spec.type_name == "bool": + return + + template = get_jinja2_template(environment, "encoder", "default_spec") + print(template.render()) + + if default_case is None or isinstance(default_case.arm, _XdrVoid): + template = get_jinja2_template(environment, "encoder", "break") + print(template.render()) + return + + assert isinstance(default_case.arm, _XdrBasic) + template = get_jinja2_template(environment, "encoder", default_case.arm.template) + print( + template.render( + name=default_case.arm.name, + type=default_case.arm.spec.type_name, + ) + ) + + +def emit_union_encoder(environment, node: _XdrUnion) -> None: + """Emit one XDR union encoder""" + template = get_jinja2_template(environment, "encoder", "open") + print(template.render(name=node.name)) + + emit_union_switch_spec_encoder(environment, node.discriminant) + + for case in node.cases: + emit_union_case_spec_encoder(environment, case) + + emit_union_default_spec_encoder(environment, node) + + template = get_jinja2_template(environment, "encoder", "close") + print(template.render()) + + +class XdrUnionGenerator(SourceGenerator): + """Generate source code for XDR unions""" + + def __init__(self, language: str, peer: str): + """Initialize an instance of this class""" + self.environment = create_jinja2_environment(language, "union") + self.peer = peer + + def emit_declaration(self, node: _XdrUnion) -> None: + """Emit one declaration pair for an XDR union""" + emit_union_declaration(self.environment, node) + + def emit_definition(self, node: _XdrUnion) -> None: + """Emit one definition for an XDR union""" + emit_union_definition(self.environment, node) + + def emit_decoder(self, node: _XdrUnion) -> None: + """Emit one decoder function for an XDR union""" + emit_union_decoder(self.environment, node) + + def emit_encoder(self, node: _XdrUnion) -> None: + """Emit one encoder function for an XDR union""" + emit_union_encoder(self.environment, node) diff --git a/tools/net/sunrpc/xdrgen/grammars/xdr.lark b/tools/net/sunrpc/xdrgen/grammars/xdr.lark new file mode 100644 index 000000000000..f3c4552e548d --- /dev/null +++ b/tools/net/sunrpc/xdrgen/grammars/xdr.lark @@ -0,0 +1,119 @@ +// A Lark grammar for the XDR specification language based on +// https://tools.ietf.org/html/rfc4506 Section 6.3 + +declaration : "opaque" identifier "[" value "]" -> fixed_length_opaque + | "opaque" identifier "<" [ value ] ">" -> variable_length_opaque + | "string" identifier "<" [ value ] ">" -> variable_length_string + | type_specifier identifier "[" value "]" -> fixed_length_array + | type_specifier identifier "<" [ value ] ">" -> variable_length_array + | type_specifier "*" identifier -> optional_data + | type_specifier identifier -> basic + | "void" -> void + +value : decimal_constant + | hexadecimal_constant + | octal_constant + | identifier + +constant : decimal_constant | hexadecimal_constant | octal_constant + +type_specifier : unsigned_hyper + | unsigned_long + | unsigned_int + | hyper + | long + | int + | float + | double + | quadruple + | bool + | enum_type_spec + | struct_type_spec + | union_type_spec + | identifier + +unsigned_hyper : "unsigned" "hyper" +unsigned_long : "unsigned" "long" +unsigned_int : "unsigned" "int" +hyper : "hyper" +long : "long" +int : "int" +float : "float" +double : "double" +quadruple : "quadruple" +bool : "bool" + +enum_type_spec : "enum" enum_body + +enum_body : "{" ( identifier "=" value ) ( "," identifier "=" value )* "}" + +struct_type_spec : "struct" struct_body + +struct_body : "{" ( declaration ";" )+ "}" + +union_type_spec : "union" union_body + +union_body : switch_spec "{" case_spec+ [ default_spec ] "}" + +switch_spec : "switch" "(" declaration ")" + +case_spec : ( "case" value ":" )+ declaration ";" + +default_spec : "default" ":" declaration ";" + +constant_def : "const" identifier "=" value ";" + +type_def : "typedef" declaration ";" -> typedef + | "enum" identifier enum_body ";" -> enum + | "struct" identifier struct_body ";" -> struct + | "union" identifier union_body ";" -> union + +specification : definition* + +definition : constant_def + | type_def + | program_def + | pragma_def + +// +// RPC program definitions not specified in RFC 4506 +// + +program_def : "program" identifier "{" version_def+ "}" "=" constant ";" + +version_def : "version" identifier "{" procedure_def+ "}" "=" constant ";" + +procedure_def : type_specifier identifier "(" type_specifier ")" "=" constant ";" + +pragma_def : "pragma" directive identifier [ identifier ] ";" + +directive : exclude_directive + | header_directive + | pages_directive + | public_directive + | skip_directive + +exclude_directive : "exclude" +header_directive : "header" +pages_directive : "pages" +public_directive : "public" +skip_directive : "skip" + +// +// XDR language primitives +// + +identifier : /([a-z]|[A-Z])(_|[a-z]|[A-Z]|[0-9])*/ + +decimal_constant : /[\+-]?(0|[1-9][0-9]*)/ +hexadecimal_constant : /0x([a-f]|[A-F]|[0-9])+/ +octal_constant : /0[0-7]+/ + +PASSTHRU : "%" | "%" /.+/ +%ignore PASSTHRU + +%import common.C_COMMENT +%ignore C_COMMENT + +%import common.WS +%ignore WS diff --git a/tools/net/sunrpc/xdrgen/subcmds/__init__.py b/tools/net/sunrpc/xdrgen/subcmds/__init__.py new file mode 100644 index 000000000000..c940e9275252 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/subcmds/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +# Just to make sphinx-apidoc document this directory diff --git a/tools/net/sunrpc/xdrgen/subcmds/declarations.py b/tools/net/sunrpc/xdrgen/subcmds/declarations.py new file mode 100644 index 000000000000..c5e8d79986ef --- /dev/null +++ b/tools/net/sunrpc/xdrgen/subcmds/declarations.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +# ex: set filetype=python: + +"""Translate an XDR specification into executable code that +can be compiled for the Linux kernel.""" + +import logging + +from argparse import Namespace +from lark import logger +from lark.exceptions import UnexpectedInput + +from generators.constant import XdrConstantGenerator +from generators.enum import XdrEnumGenerator +from generators.header_bottom import XdrHeaderBottomGenerator +from generators.header_top import XdrHeaderTopGenerator +from generators.pointer import XdrPointerGenerator +from generators.program import XdrProgramGenerator +from generators.typedef import XdrTypedefGenerator +from generators.struct import XdrStructGenerator +from generators.union import XdrUnionGenerator + +from xdr_ast import transform_parse_tree, _RpcProgram, Specification +from xdr_ast import _XdrConstant, _XdrEnum, _XdrPointer +from xdr_ast import _XdrTypedef, _XdrStruct, _XdrUnion +from xdr_parse import xdr_parser, set_xdr_annotate + +logger.setLevel(logging.INFO) + + +def emit_header_declarations( + root: Specification, language: str, peer: str +) -> None: + """Emit header declarations""" + for definition in root.definitions: + if isinstance(definition.value, _XdrEnum): + gen = XdrEnumGenerator(language, peer) + elif isinstance(definition.value, _XdrPointer): + gen = XdrPointerGenerator(language, peer) + elif isinstance(definition.value, _XdrTypedef): + gen = XdrTypedefGenerator(language, peer) + elif isinstance(definition.value, _XdrStruct): + gen = XdrStructGenerator(language, peer) + elif isinstance(definition.value, _XdrUnion): + gen = XdrUnionGenerator(language, peer) + elif isinstance(definition.value, _RpcProgram): + gen = XdrProgramGenerator(language, peer) + else: + continue + gen.emit_declaration(definition.value) + + +def handle_parse_error(e: UnexpectedInput) -> bool: + """Simple parse error reporting, no recovery attempted""" + print(e) + return True + + +def subcmd(args: Namespace) -> int: + """Generate definitions and declarations""" + + set_xdr_annotate(args.annotate) + parser = xdr_parser() + with open(args.filename, encoding="utf-8") as f: + parse_tree = parser.parse(f.read(), on_error=handle_parse_error) + ast = transform_parse_tree(parse_tree) + + gen = XdrHeaderTopGenerator(args.language, args.peer) + gen.emit_declaration(args.filename, ast) + + emit_header_declarations(ast, args.language, args.peer) + + gen = XdrHeaderBottomGenerator(args.language, args.peer) + gen.emit_declaration(args.filename, ast) + + return 0 diff --git a/tools/net/sunrpc/xdrgen/subcmds/definitions.py b/tools/net/sunrpc/xdrgen/subcmds/definitions.py new file mode 100644 index 000000000000..5cd13d53221f --- /dev/null +++ b/tools/net/sunrpc/xdrgen/subcmds/definitions.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +# ex: set filetype=python: + +"""Translate an XDR specification into executable code that +can be compiled for the Linux kernel.""" + +import logging + +from argparse import Namespace +from lark import logger +from lark.exceptions import UnexpectedInput + +from generators.constant import XdrConstantGenerator +from generators.enum import XdrEnumGenerator +from generators.header_bottom import XdrHeaderBottomGenerator +from generators.header_top import XdrHeaderTopGenerator +from generators.pointer import XdrPointerGenerator +from generators.program import XdrProgramGenerator +from generators.typedef import XdrTypedefGenerator +from generators.struct import XdrStructGenerator +from generators.union import XdrUnionGenerator + +from xdr_ast import transform_parse_tree, Specification +from xdr_ast import _RpcProgram, _XdrConstant, _XdrEnum, _XdrPointer +from xdr_ast import _XdrTypedef, _XdrStruct, _XdrUnion +from xdr_parse import xdr_parser, set_xdr_annotate + +logger.setLevel(logging.INFO) + + +def emit_header_definitions( + root: Specification, language: str, peer: str +) -> None: + """Emit header definitions""" + for definition in root.definitions: + if isinstance(definition.value, _XdrConstant): + gen = XdrConstantGenerator(language, peer) + elif isinstance(definition.value, _XdrEnum): + gen = XdrEnumGenerator(language, peer) + elif isinstance(definition.value, _XdrPointer): + gen = XdrPointerGenerator(language, peer) + elif isinstance(definition.value, _RpcProgram): + gen = XdrProgramGenerator(language, peer) + elif isinstance(definition.value, _XdrTypedef): + gen = XdrTypedefGenerator(language, peer) + elif isinstance(definition.value, _XdrStruct): + gen = XdrStructGenerator(language, peer) + elif isinstance(definition.value, _XdrUnion): + gen = XdrUnionGenerator(language, peer) + else: + continue + gen.emit_definition(definition.value) + + +def handle_parse_error(e: UnexpectedInput) -> bool: + """Simple parse error reporting, no recovery attempted""" + print(e) + return True + + +def subcmd(args: Namespace) -> int: + """Generate definitions""" + + set_xdr_annotate(args.annotate) + parser = xdr_parser() + with open(args.filename, encoding="utf-8") as f: + parse_tree = parser.parse(f.read(), on_error=handle_parse_error) + ast = transform_parse_tree(parse_tree) + + gen = XdrHeaderTopGenerator(args.language, args.peer) + gen.emit_definition(args.filename, ast) + + emit_header_definitions(ast, args.language, args.peer) + + gen = XdrHeaderBottomGenerator(args.language, args.peer) + gen.emit_definition(args.filename, ast) + + return 0 diff --git a/tools/net/sunrpc/xdrgen/subcmds/lint.py b/tools/net/sunrpc/xdrgen/subcmds/lint.py new file mode 100644 index 000000000000..36cc43717d30 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/subcmds/lint.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# ex: set filetype=python: + +"""Translate an XDR specification into executable code that +can be compiled for the Linux kernel.""" + +import logging + +from argparse import Namespace +from lark import logger +from lark.exceptions import UnexpectedInput + +from xdr_parse import xdr_parser +from xdr_ast import transform_parse_tree + +logger.setLevel(logging.DEBUG) + + +def handle_parse_error(e: UnexpectedInput) -> bool: + """Simple parse error reporting, no recovery attempted""" + print(e) + return True + + +def subcmd(args: Namespace) -> int: + """Lexical and syntax check of an XDR specification""" + + parser = xdr_parser() + with open(args.filename, encoding="utf-8") as f: + parse_tree = parser.parse(f.read(), on_error=handle_parse_error) + transform_parse_tree(parse_tree) + + return 0 diff --git a/tools/net/sunrpc/xdrgen/subcmds/source.py b/tools/net/sunrpc/xdrgen/subcmds/source.py new file mode 100644 index 000000000000..00c04ad15b89 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/subcmds/source.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +# ex: set filetype=python: + +"""Translate an XDR specification into executable code that +can be compiled for the Linux kernel.""" + +import logging + +from argparse import Namespace +from lark import logger +from lark.exceptions import UnexpectedInput + +from generators.source_top import XdrSourceTopGenerator +from generators.enum import XdrEnumGenerator +from generators.pointer import XdrPointerGenerator +from generators.program import XdrProgramGenerator +from generators.typedef import XdrTypedefGenerator +from generators.struct import XdrStructGenerator +from generators.union import XdrUnionGenerator + +from xdr_ast import transform_parse_tree, _RpcProgram, Specification +from xdr_ast import _XdrAst, _XdrEnum, _XdrPointer +from xdr_ast import _XdrStruct, _XdrTypedef, _XdrUnion + +from xdr_parse import xdr_parser, set_xdr_annotate + +logger.setLevel(logging.INFO) + + +def emit_source_decoder(node: _XdrAst, language: str, peer: str) -> None: + """Emit one XDR decoder function for a source file""" + if isinstance(node, _XdrEnum): + gen = XdrEnumGenerator(language, peer) + elif isinstance(node, _XdrPointer): + gen = XdrPointerGenerator(language, peer) + elif isinstance(node, _XdrTypedef): + gen = XdrTypedefGenerator(language, peer) + elif isinstance(node, _XdrStruct): + gen = XdrStructGenerator(language, peer) + elif isinstance(node, _XdrUnion): + gen = XdrUnionGenerator(language, peer) + elif isinstance(node, _RpcProgram): + gen = XdrProgramGenerator(language, peer) + else: + return + gen.emit_decoder(node) + + +def emit_source_encoder(node: _XdrAst, language: str, peer: str) -> None: + """Emit one XDR encoder function for a source file""" + if isinstance(node, _XdrEnum): + gen = XdrEnumGenerator(language, peer) + elif isinstance(node, _XdrPointer): + gen = XdrPointerGenerator(language, peer) + elif isinstance(node, _XdrTypedef): + gen = XdrTypedefGenerator(language, peer) + elif isinstance(node, _XdrStruct): + gen = XdrStructGenerator(language, peer) + elif isinstance(node, _XdrUnion): + gen = XdrUnionGenerator(language, peer) + elif isinstance(node, _RpcProgram): + gen = XdrProgramGenerator(language, peer) + else: + return + gen.emit_encoder(node) + + +def generate_server_source(filename: str, root: Specification, language: str) -> None: + """Generate server-side source code""" + + gen = XdrSourceTopGenerator(language, "server") + gen.emit_source(filename, root) + + for definition in root.definitions: + emit_source_decoder(definition.value, language, "server") + for definition in root.definitions: + emit_source_encoder(definition.value, language, "server") + + +def generate_client_source(filename: str, root: Specification, language: str) -> None: + """Generate server-side source code""" + + gen = XdrSourceTopGenerator(language, "client") + gen.emit_source(filename, root) + + # cel: todo: client needs XDR size macros + + for definition in root.definitions: + emit_source_encoder(definition.value, language, "client") + for definition in root.definitions: + emit_source_decoder(definition.value, language, "client") + + # cel: todo: client needs PROC macros + + +def handle_parse_error(e: UnexpectedInput) -> bool: + """Simple parse error reporting, no recovery attempted""" + print(e) + return True + + +def subcmd(args: Namespace) -> int: + """Generate encoder and decoder functions""" + + set_xdr_annotate(args.annotate) + parser = xdr_parser() + with open(args.filename, encoding="utf-8") as f: + parse_tree = parser.parse(f.read(), on_error=handle_parse_error) + ast = transform_parse_tree(parse_tree) + match args.peer: + case "server": + generate_server_source(args.filename, ast, args.language) + case "client": + generate_client_source(args.filename, ast, args.language) + case _: + print("Code generation for", args.peer, "is not yet supported") + + return 0 diff --git a/tools/net/sunrpc/xdrgen/templates/C/constants/definition.j2 b/tools/net/sunrpc/xdrgen/templates/C/constants/definition.j2 new file mode 100644 index 000000000000..d648ca4193f8 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/constants/definition.j2 @@ -0,0 +1,3 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +enum { {{ name }} = {{ value }} }; diff --git a/tools/net/sunrpc/xdrgen/templates/C/enum/declaration/close.j2 b/tools/net/sunrpc/xdrgen/templates/C/enum/declaration/close.j2 new file mode 100644 index 000000000000..ab1e576c9531 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/enum/declaration/close.j2 @@ -0,0 +1,4 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +bool xdrgen_decode_{{ name }}(struct xdr_stream *xdr, enum {{ name }} *ptr); +bool xdrgen_encode_{{ name }}(struct xdr_stream *xdr, enum {{ name }} value); diff --git a/tools/net/sunrpc/xdrgen/templates/C/enum/decoder/enum.j2 b/tools/net/sunrpc/xdrgen/templates/C/enum/decoder/enum.j2 new file mode 100644 index 000000000000..341d829afeda --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/enum/decoder/enum.j2 @@ -0,0 +1,19 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* enum {{ name }} */ +{% endif %} +{% if name in public_apis %} +bool +{% else %} +static bool __maybe_unused +{% endif %} +xdrgen_decode_{{ name }}(struct xdr_stream *xdr, enum {{ name }} *ptr) +{ + u32 val; + + if (xdr_stream_decode_u32(xdr, &val) < 0) + return false; + *ptr = val; + return true; +} diff --git a/tools/net/sunrpc/xdrgen/templates/C/enum/definition/close.j2 b/tools/net/sunrpc/xdrgen/templates/C/enum/definition/close.j2 new file mode 100644 index 000000000000..9e62344a976a --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/enum/definition/close.j2 @@ -0,0 +1,2 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/enum/definition/enumerator.j2 b/tools/net/sunrpc/xdrgen/templates/C/enum/definition/enumerator.j2 new file mode 100644 index 000000000000..ff0b893b8b14 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/enum/definition/enumerator.j2 @@ -0,0 +1,2 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + {{ name }} = {{ value }}, diff --git a/tools/net/sunrpc/xdrgen/templates/C/enum/definition/open.j2 b/tools/net/sunrpc/xdrgen/templates/C/enum/definition/open.j2 new file mode 100644 index 000000000000..b25335221d48 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/enum/definition/open.j2 @@ -0,0 +1,3 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +enum {{ name }} { diff --git a/tools/net/sunrpc/xdrgen/templates/C/enum/encoder/enum.j2 b/tools/net/sunrpc/xdrgen/templates/C/enum/encoder/enum.j2 new file mode 100644 index 000000000000..bd0a770e50f2 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/enum/encoder/enum.j2 @@ -0,0 +1,14 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* enum {{ name }} */ +{% endif %} +{% if name in public_apis %} +bool +{% else %} +static bool __maybe_unused +{% endif %} +xdrgen_encode_{{ name }}(struct xdr_stream *xdr, enum {{ name }} value) +{ + return xdr_stream_encode_u32(xdr, value) == XDR_UNIT; +} diff --git a/tools/net/sunrpc/xdrgen/templates/C/header_bottom/declaration/header.j2 b/tools/net/sunrpc/xdrgen/templates/C/header_bottom/declaration/header.j2 new file mode 100644 index 000000000000..0bb8c6fc0c20 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/header_bottom/declaration/header.j2 @@ -0,0 +1,3 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +#endif /* _LINUX_XDRGEN_{{ infix }}_DECL_H */ diff --git a/tools/net/sunrpc/xdrgen/templates/C/header_bottom/definition/header.j2 b/tools/net/sunrpc/xdrgen/templates/C/header_bottom/definition/header.j2 new file mode 100644 index 000000000000..69069d08dc91 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/header_bottom/definition/header.j2 @@ -0,0 +1,3 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +#endif /* _LINUX_XDRGEN_{{ infix }}_DEF_H */ diff --git a/tools/net/sunrpc/xdrgen/templates/C/header_top/declaration/header.j2 b/tools/net/sunrpc/xdrgen/templates/C/header_top/declaration/header.j2 new file mode 100644 index 000000000000..ebb4e1d32f85 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/header_top/declaration/header.j2 @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Generated by xdrgen. Manual edits will be lost. */ +/* XDR specification file: {{ filename }} */ +/* XDR specification modification time: {{ mtime }} */ + +#ifndef _LINUX_XDRGEN_{{ infix }}_DECL_H +#define _LINUX_XDRGEN_{{ infix }}_DECL_H + +#include + +#include +#include +#include +#include diff --git a/tools/net/sunrpc/xdrgen/templates/C/header_top/definition/header.j2 b/tools/net/sunrpc/xdrgen/templates/C/header_top/definition/header.j2 new file mode 100644 index 000000000000..92f1fd4ba024 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/header_top/definition/header.j2 @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Generated by xdrgen. Manual edits will be lost. */ +/* XDR specification file: {{ filename }} */ +/* XDR specification modification time: {{ mtime }} */ + +#ifndef _LINUX_XDRGEN_{{ infix }}_DEF_H +#define _LINUX_XDRGEN_{{ infix }}_DEF_H + +#include +#include diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/declaration/close.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/declaration/close.j2 new file mode 100644 index 000000000000..816291184e8c --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/declaration/close.j2 @@ -0,0 +1,4 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +bool xdrgen_decode_{{ name }}(struct xdr_stream *xdr, struct {{ name }} *ptr); +bool xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const struct {{ name }} *value); diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/basic.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/basic.j2 new file mode 100644 index 000000000000..cde4ab53f4be --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/basic.j2 @@ -0,0 +1,6 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (basic) */ +{% endif %} + if (!xdrgen_decode_{{ type }}(xdr, &ptr->{{ name }})) + return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/close.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/close.j2 new file mode 100644 index 000000000000..5bf010665f84 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/close.j2 @@ -0,0 +1,3 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + return true; +}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/fixed_length_array.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/fixed_length_array.j2 new file mode 100644 index 000000000000..cfd64217ad82 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/fixed_length_array.j2 @@ -0,0 +1,8 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (fixed-length array) */ +{% endif %} + for (u32 i = 0; i < {{ size }}; i++) { + if (xdrgen_decode_{{ type }}(xdr, &ptr->{{ name }}.items[i]) < 0) + return false; + } diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/fixed_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/fixed_length_opaque.j2 new file mode 100644 index 000000000000..b4695ece1884 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/fixed_length_opaque.j2 @@ -0,0 +1,6 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (fixed-length opaque) */ +{% endif %} + if (xdr_stream_decode_opaque_fixed(xdr, ptr->{{ name }}, {{ size }}) < 0) + return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/open.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/open.j2 new file mode 100644 index 000000000000..c093d9e3c9ad --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/open.j2 @@ -0,0 +1,22 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* pointer {{ name }} */ +{% endif %} +{% if name in public_apis %} +bool +{% else %} +static bool __maybe_unused +{% endif %} +xdrgen_decode_{{ name }}(struct xdr_stream *xdr, struct {{ name }} *ptr) +{ + bool opted; + +{% if annotate %} + /* opted */ +{% endif %} + if (!xdrgen_decode_bool(xdr, &opted)) + return false; + if (!opted) + return true; + diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/optional_data.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/optional_data.j2 new file mode 100644 index 000000000000..b6834299a04b --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/optional_data.j2 @@ -0,0 +1,6 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (optional data) */ +{% endif %} + if (!xdrgen_decode_{{ type }}(xdr, ptr->{{ name }})) + return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/variable_length_array.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/variable_length_array.j2 new file mode 100644 index 000000000000..f54ccd136762 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/variable_length_array.j2 @@ -0,0 +1,13 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (variable-length array) */ +{% endif %} + if (xdr_stream_decode_u32(xdr, &ptr->{{ name }}.count) != XDR_UNIT) + return false; +{% if maxsize != "0" %} + if (ptr->{{ name }}.count > {{ maxsize }}) + return false; +{% endif %} + for (u32 i = 0; i < ptr->{{ name }}.count; i++) + if (!xdrgen_decode_{{ type }}(xdr, &ptr->{{ name }}.element[i])) + return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/variable_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/variable_length_opaque.j2 new file mode 100644 index 000000000000..9a814de54ae8 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/variable_length_opaque.j2 @@ -0,0 +1,6 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (variable-length opaque) */ +{% endif %} + if (!xdrgen_decode_opaque(xdr, (opaque *)ptr, {{ maxsize }})) + return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/variable_length_string.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/variable_length_string.j2 new file mode 100644 index 000000000000..12d20b143b43 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/variable_length_string.j2 @@ -0,0 +1,6 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (variable-length string) */ +{% endif %} + if (!xdrgen_decode_string(xdr, (string *)ptr, {{ maxsize }})) + return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/definition/basic.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/definition/basic.j2 new file mode 100644 index 000000000000..b3430895f311 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/definition/basic.j2 @@ -0,0 +1,5 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* (basic) */ +{% endif %} + {{ classifier }}{{ type }} {{ name }}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/definition/close.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/definition/close.j2 new file mode 100644 index 000000000000..9e62344a976a --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/definition/close.j2 @@ -0,0 +1,2 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/definition/fixed_length_array.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/definition/fixed_length_array.j2 new file mode 100644 index 000000000000..66be836826a0 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/definition/fixed_length_array.j2 @@ -0,0 +1,5 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* (fixed-length array) */ +{% endif %} + {{ type }} {{ name }}[{{ size }}]; diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/definition/fixed_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/definition/fixed_length_opaque.j2 new file mode 100644 index 000000000000..0daba19aa0f0 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/definition/fixed_length_opaque.j2 @@ -0,0 +1,5 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* (fixed-length opaque) */ +{% endif %} + u8 {{ name }}[{{ size }}]; diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/definition/open.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/definition/open.j2 new file mode 100644 index 000000000000..bc886b818d85 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/definition/open.j2 @@ -0,0 +1,6 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* pointer {{ name }} */ +{% endif %} +struct {{ name }} { diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/definition/optional_data.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/definition/optional_data.j2 new file mode 100644 index 000000000000..a33341f45e8f --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/definition/optional_data.j2 @@ -0,0 +1,5 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* (optional data) */ +{% endif %} + {{ classifier }}{{ type }} *{{ name }}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/definition/variable_length_array.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/definition/variable_length_array.j2 new file mode 100644 index 000000000000..5d767f9b3674 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/definition/variable_length_array.j2 @@ -0,0 +1,8 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* (variable-length array) */ +{% endif %} + struct { + u32 count; + {{ classifier }}{{ type }} *element; + } {{ name }}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/definition/variable_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/definition/variable_length_opaque.j2 new file mode 100644 index 000000000000..4d0cd84be3db --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/definition/variable_length_opaque.j2 @@ -0,0 +1,5 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* (variable-length opaque) */ +{% endif %} + opaque {{ name }}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/definition/variable_length_string.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/definition/variable_length_string.j2 new file mode 100644 index 000000000000..2de2feec77db --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/definition/variable_length_string.j2 @@ -0,0 +1,5 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* (variable-length string) */ +{% endif %} + string {{ name }}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/basic.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/basic.j2 new file mode 100644 index 000000000000..a7d3695c5a6a --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/basic.j2 @@ -0,0 +1,10 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (basic) */ +{% endif %} +{% if type in pass_by_reference %} + if (!xdrgen_encode_{{ type }}(xdr, &value->{{ name }})) +{% else %} + if (!xdrgen_encode_{{ type }}(xdr, value->{{ name }})) +{% endif %} + return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/close.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/close.j2 new file mode 100644 index 000000000000..5bf010665f84 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/close.j2 @@ -0,0 +1,3 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + return true; +}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/fixed_length_array.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/fixed_length_array.j2 new file mode 100644 index 000000000000..b01833a2c7a1 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/fixed_length_array.j2 @@ -0,0 +1,12 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (fixed-length array) */ +{% endif %} + for (u32 i = 0; i < {{ size }}; i++) { +{% if type in pass_by_reference %} + if (xdrgen_encode_{{ type }}(xdr, &value->items[i]) < 0) +{% else %} + if (xdrgen_encode_{{ type }}(xdr, value->items[i]) < 0) +{% endif %} + return false; + } diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/fixed_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/fixed_length_opaque.j2 new file mode 100644 index 000000000000..07bc91919898 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/fixed_length_opaque.j2 @@ -0,0 +1,6 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (fixed-length opaque) */ +{% endif %} + if (xdr_stream_encode_opaque_fixed(xdr, value->{{ name }}, {{ size }}) < 0) + return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/open.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/open.j2 new file mode 100644 index 000000000000..d67fae200261 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/open.j2 @@ -0,0 +1,20 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* pointer {{ name }} */ +{% endif %} +{% if name in public_apis %} +bool +{% else %} +static bool __maybe_unused +{% endif %} +xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const struct {{ name }} *value) +{ +{% if annotate %} + /* opted */ +{% endif %} + if (!xdrgen_encode_bool(xdr, value != NULL)) + return false; + if (!value) + return true; + diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/optional_data.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/optional_data.j2 new file mode 100644 index 000000000000..16fb3e09bba1 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/optional_data.j2 @@ -0,0 +1,6 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (optional data) */ +{% endif %} + if (!xdrgen_encode_{{ type }}(xdr, value->{{ name }})) + return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/variable_length_array.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/variable_length_array.j2 new file mode 100644 index 000000000000..0ec8660d621a --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/variable_length_array.j2 @@ -0,0 +1,15 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (variable-length array) */ +{% endif %} + if (value->{{ name }}.count > {{ maxsize }}) + return false; + if (xdr_stream_encode_u32(xdr, value->{{ name }}.count) != XDR_UNIT) + return false; + for (u32 i = 0; i < value->{{ name }}.count; i++) +{% if type in pass_by_reference %} + if (!xdrgen_encode_{{ type }}(xdr, &value->{{ name }}.element[i])) +{% else %} + if (!xdrgen_encode_{{ type }}(xdr, value->{{ name }}.element[i])) +{% endif %} + return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/variable_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/variable_length_opaque.j2 new file mode 100644 index 000000000000..1d477c2d197a --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/variable_length_opaque.j2 @@ -0,0 +1,8 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (variable-length opaque) */ +{% endif %} + if (value->{{ name }}.len > {{ maxsize }}) + return false; + if (xdr_stream_encode_opaque(xdr, value->{{ name }}.data, value->{{ name }}.len) < 0) + return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/variable_length_string.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/variable_length_string.j2 new file mode 100644 index 000000000000..cf65b71eaef3 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/variable_length_string.j2 @@ -0,0 +1,8 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (variable-length string) */ +{% endif %} + if (value->{{ name }}.len > {{ maxsize }}) + return false; + if (xdr_stream_encode_opaque(xdr, value->{{ name }}.data, value->{{ name }}.len) < 0) + return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/program/declaration/argument.j2 b/tools/net/sunrpc/xdrgen/templates/C/program/declaration/argument.j2 new file mode 100644 index 000000000000..4364fed19162 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/program/declaration/argument.j2 @@ -0,0 +1,2 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +bool {{ program }}_svc_decode_{{ argument }}(struct svc_rqst *rqstp, struct xdr_stream *xdr); diff --git a/tools/net/sunrpc/xdrgen/templates/C/program/declaration/result.j2 b/tools/net/sunrpc/xdrgen/templates/C/program/declaration/result.j2 new file mode 100644 index 000000000000..e0ea1e849910 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/program/declaration/result.j2 @@ -0,0 +1,2 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +bool {{ program }}_svc_encode_{{ result }}(struct svc_rqst *rqstp, struct xdr_stream *xdr); diff --git a/tools/net/sunrpc/xdrgen/templates/C/program/decoder/argument.j2 b/tools/net/sunrpc/xdrgen/templates/C/program/decoder/argument.j2 new file mode 100644 index 000000000000..0b1709cca0d4 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/program/decoder/argument.j2 @@ -0,0 +1,21 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +/** + * {{ program }}_svc_decode_{{ argument }} - Decode a {{ argument }} argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool {{ program }}_svc_decode_{{ argument }}(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ +{% if argument == 'void' %} + return xdrgen_decode_void(xdr); +{% else %} + struct {{ argument }} *argp = rqstp->rq_argp; + + return xdrgen_decode_{{ argument }}(xdr, argp); +{% endif %} +} diff --git a/tools/net/sunrpc/xdrgen/templates/C/program/decoder/result.j2 b/tools/net/sunrpc/xdrgen/templates/C/program/decoder/result.j2 new file mode 100644 index 000000000000..d304eccb5c40 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/program/decoder/result.j2 @@ -0,0 +1,22 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* Decode {{ result }} results */ +{% endif %} +static int {{ program }}_xdr_dec_{{ result }}(struct rpc_rqst *req, + struct xdr_stream *xdr, void *data) +{ +{% if result == 'void' %} + xdrgen_decode_void(xdr); +{% else %} + struct {{ result }} *result = data; + + if (!xdrgen_decode_{{ result }}(xdr, result)) + return -EIO; + if (result->stat != nfs_ok) { + trace_nfs_xdr_status(xdr, (int)result->stat); + return {{ program }}_stat_to_errno(result->stat); + } +{% endif %} + return 0; +} diff --git a/tools/net/sunrpc/xdrgen/templates/C/program/definition/close.j2 b/tools/net/sunrpc/xdrgen/templates/C/program/definition/close.j2 new file mode 100644 index 000000000000..9e62344a976a --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/program/definition/close.j2 @@ -0,0 +1,2 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/program/definition/open.j2 b/tools/net/sunrpc/xdrgen/templates/C/program/definition/open.j2 new file mode 100644 index 000000000000..f9a6d439f156 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/program/definition/open.j2 @@ -0,0 +1,6 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* procedure numbers for {{ program }} */ +{% endif %} +enum { diff --git a/tools/net/sunrpc/xdrgen/templates/C/program/definition/procedure.j2 b/tools/net/sunrpc/xdrgen/templates/C/program/definition/procedure.j2 new file mode 100644 index 000000000000..ff0b893b8b14 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/program/definition/procedure.j2 @@ -0,0 +1,2 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + {{ name }} = {{ value }}, diff --git a/tools/net/sunrpc/xdrgen/templates/C/program/encoder/argument.j2 b/tools/net/sunrpc/xdrgen/templates/C/program/encoder/argument.j2 new file mode 100644 index 000000000000..2fbb5bd13aec --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/program/encoder/argument.j2 @@ -0,0 +1,16 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{%if annotate %} +/* Encode {{ argument }} arguments */ +{% endif %} +static void {{ program }}_xdr_enc_{{ argument }}(struct rpc_rqst *req, + struct xdr_stream *xdr, const void *data) +{ +{% if argument == 'void' %} + xdrgen_encode_void(xdr); +{% else %} + const struct {{ argument }} *args = data; + + xdrgen_encode_{{ argument }}(xdr, args); +{% endif %} +} diff --git a/tools/net/sunrpc/xdrgen/templates/C/program/encoder/result.j2 b/tools/net/sunrpc/xdrgen/templates/C/program/encoder/result.j2 new file mode 100644 index 000000000000..6fc61a5d47b7 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/program/encoder/result.j2 @@ -0,0 +1,21 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +/** + * {{ program }}_svc_encode_{{ result }} - Encode a {{ result }} result + * @rqstp: RPC transaction context + * @xdr: target XDR data stream + * + * Return values: + * %true: procedure results encoded successfully + * %false: encode failed + */ +bool {{ program }}_svc_encode_{{ result }}(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ +{% if result == 'void' %} + return xdrgen_encode_void(xdr); +{% else %} + struct {{ result }} *resp = rqstp->rq_resp; + + return xdrgen_encode_{{ result }}(xdr, resp); +{% endif %} +} diff --git a/tools/net/sunrpc/xdrgen/templates/C/source_top/client.j2 b/tools/net/sunrpc/xdrgen/templates/C/source_top/client.j2 new file mode 100644 index 000000000000..e3a802cbc4d7 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/source_top/client.j2 @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 +// Generated by xdrgen. Manual edits will be lost. +// XDR specification file: {{ filename }} +// XDR specification modification time: {{ mtime }} + +#include + +#include "{{ program }}xdr_gen.h" diff --git a/tools/net/sunrpc/xdrgen/templates/C/source_top/server.j2 b/tools/net/sunrpc/xdrgen/templates/C/source_top/server.j2 new file mode 100644 index 000000000000..974e1d971e5d --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/source_top/server.j2 @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 +// Generated by xdrgen. Manual edits will be lost. +// XDR specification file: {{ filename }} +// XDR specification modification time: {{ mtime }} + +#include + +#include "{{ program }}xdr_gen.h" diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/declaration/close.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/declaration/close.j2 new file mode 100644 index 000000000000..816291184e8c --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/declaration/close.j2 @@ -0,0 +1,4 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +bool xdrgen_decode_{{ name }}(struct xdr_stream *xdr, struct {{ name }} *ptr); +bool xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const struct {{ name }} *value); diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/basic.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/basic.j2 new file mode 100644 index 000000000000..cde4ab53f4be --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/basic.j2 @@ -0,0 +1,6 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (basic) */ +{% endif %} + if (!xdrgen_decode_{{ type }}(xdr, &ptr->{{ name }})) + return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/close.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/close.j2 new file mode 100644 index 000000000000..5bf010665f84 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/close.j2 @@ -0,0 +1,3 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + return true; +}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/fixed_length_array.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/fixed_length_array.j2 new file mode 100644 index 000000000000..cfd64217ad82 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/fixed_length_array.j2 @@ -0,0 +1,8 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (fixed-length array) */ +{% endif %} + for (u32 i = 0; i < {{ size }}; i++) { + if (xdrgen_decode_{{ type }}(xdr, &ptr->{{ name }}.items[i]) < 0) + return false; + } diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/fixed_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/fixed_length_opaque.j2 new file mode 100644 index 000000000000..b4695ece1884 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/fixed_length_opaque.j2 @@ -0,0 +1,6 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (fixed-length opaque) */ +{% endif %} + if (xdr_stream_decode_opaque_fixed(xdr, ptr->{{ name }}, {{ size }}) < 0) + return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/open.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/open.j2 new file mode 100644 index 000000000000..289e67259f55 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/open.j2 @@ -0,0 +1,12 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* struct {{ name }} */ +{% endif %} +{% if name in public_apis %} +bool +{% else %} +static bool __maybe_unused +{% endif %} +xdrgen_decode_{{ name }}(struct xdr_stream *xdr, struct {{ name }} *ptr) +{ diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/optional_data.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/optional_data.j2 new file mode 100644 index 000000000000..b6834299a04b --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/optional_data.j2 @@ -0,0 +1,6 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (optional data) */ +{% endif %} + if (!xdrgen_decode_{{ type }}(xdr, ptr->{{ name }})) + return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/variable_length_array.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/variable_length_array.j2 new file mode 100644 index 000000000000..f54ccd136762 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/variable_length_array.j2 @@ -0,0 +1,13 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (variable-length array) */ +{% endif %} + if (xdr_stream_decode_u32(xdr, &ptr->{{ name }}.count) != XDR_UNIT) + return false; +{% if maxsize != "0" %} + if (ptr->{{ name }}.count > {{ maxsize }}) + return false; +{% endif %} + for (u32 i = 0; i < ptr->{{ name }}.count; i++) + if (!xdrgen_decode_{{ type }}(xdr, &ptr->{{ name }}.element[i])) + return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/variable_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/variable_length_opaque.j2 new file mode 100644 index 000000000000..9a814de54ae8 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/variable_length_opaque.j2 @@ -0,0 +1,6 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (variable-length opaque) */ +{% endif %} + if (!xdrgen_decode_opaque(xdr, (opaque *)ptr, {{ maxsize }})) + return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/variable_length_string.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/variable_length_string.j2 new file mode 100644 index 000000000000..12d20b143b43 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/variable_length_string.j2 @@ -0,0 +1,6 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (variable-length string) */ +{% endif %} + if (!xdrgen_decode_string(xdr, (string *)ptr, {{ maxsize }})) + return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/definition/basic.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/definition/basic.j2 new file mode 100644 index 000000000000..b3430895f311 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/definition/basic.j2 @@ -0,0 +1,5 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* (basic) */ +{% endif %} + {{ classifier }}{{ type }} {{ name }}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/definition/close.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/definition/close.j2 new file mode 100644 index 000000000000..9e62344a976a --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/definition/close.j2 @@ -0,0 +1,2 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/definition/fixed_length_array.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/definition/fixed_length_array.j2 new file mode 100644 index 000000000000..66be836826a0 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/definition/fixed_length_array.j2 @@ -0,0 +1,5 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* (fixed-length array) */ +{% endif %} + {{ type }} {{ name }}[{{ size }}]; diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/definition/fixed_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/definition/fixed_length_opaque.j2 new file mode 100644 index 000000000000..0daba19aa0f0 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/definition/fixed_length_opaque.j2 @@ -0,0 +1,5 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* (fixed-length opaque) */ +{% endif %} + u8 {{ name }}[{{ size }}]; diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/definition/open.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/definition/open.j2 new file mode 100644 index 000000000000..07cbf5424546 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/definition/open.j2 @@ -0,0 +1,6 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* struct {{ name }} */ +{% endif %} +struct {{ name }} { diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/definition/optional_data.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/definition/optional_data.j2 new file mode 100644 index 000000000000..a33341f45e8f --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/definition/optional_data.j2 @@ -0,0 +1,5 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* (optional data) */ +{% endif %} + {{ classifier }}{{ type }} *{{ name }}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/definition/variable_length_array.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/definition/variable_length_array.j2 new file mode 100644 index 000000000000..5d767f9b3674 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/definition/variable_length_array.j2 @@ -0,0 +1,8 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* (variable-length array) */ +{% endif %} + struct { + u32 count; + {{ classifier }}{{ type }} *element; + } {{ name }}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/definition/variable_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/definition/variable_length_opaque.j2 new file mode 100644 index 000000000000..4d0cd84be3db --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/definition/variable_length_opaque.j2 @@ -0,0 +1,5 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* (variable-length opaque) */ +{% endif %} + opaque {{ name }}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/definition/variable_length_string.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/definition/variable_length_string.j2 new file mode 100644 index 000000000000..2de2feec77db --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/definition/variable_length_string.j2 @@ -0,0 +1,5 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* (variable-length string) */ +{% endif %} + string {{ name }}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/basic.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/basic.j2 new file mode 100644 index 000000000000..a7d3695c5a6a --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/basic.j2 @@ -0,0 +1,10 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (basic) */ +{% endif %} +{% if type in pass_by_reference %} + if (!xdrgen_encode_{{ type }}(xdr, &value->{{ name }})) +{% else %} + if (!xdrgen_encode_{{ type }}(xdr, value->{{ name }})) +{% endif %} + return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/close.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/close.j2 new file mode 100644 index 000000000000..5bf010665f84 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/close.j2 @@ -0,0 +1,3 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + return true; +}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/fixed_length_array.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/fixed_length_array.j2 new file mode 100644 index 000000000000..b01833a2c7a1 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/fixed_length_array.j2 @@ -0,0 +1,12 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (fixed-length array) */ +{% endif %} + for (u32 i = 0; i < {{ size }}; i++) { +{% if type in pass_by_reference %} + if (xdrgen_encode_{{ type }}(xdr, &value->items[i]) < 0) +{% else %} + if (xdrgen_encode_{{ type }}(xdr, value->items[i]) < 0) +{% endif %} + return false; + } diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/fixed_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/fixed_length_opaque.j2 new file mode 100644 index 000000000000..07bc91919898 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/fixed_length_opaque.j2 @@ -0,0 +1,6 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (fixed-length opaque) */ +{% endif %} + if (xdr_stream_encode_opaque_fixed(xdr, value->{{ name }}, {{ size }}) < 0) + return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/open.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/open.j2 new file mode 100644 index 000000000000..2286a3adf82a --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/open.j2 @@ -0,0 +1,12 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* struct {{ name }} */ +{% endif %} +{% if name in public_apis %} +bool +{% else %} +static bool __maybe_unused +{% endif %} +xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const struct {{ name }} *value) +{ diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/optional_data.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/optional_data.j2 new file mode 100644 index 000000000000..16fb3e09bba1 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/optional_data.j2 @@ -0,0 +1,6 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (optional data) */ +{% endif %} + if (!xdrgen_encode_{{ type }}(xdr, value->{{ name }})) + return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/variable_length_array.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/variable_length_array.j2 new file mode 100644 index 000000000000..0ec8660d621a --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/variable_length_array.j2 @@ -0,0 +1,15 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (variable-length array) */ +{% endif %} + if (value->{{ name }}.count > {{ maxsize }}) + return false; + if (xdr_stream_encode_u32(xdr, value->{{ name }}.count) != XDR_UNIT) + return false; + for (u32 i = 0; i < value->{{ name }}.count; i++) +{% if type in pass_by_reference %} + if (!xdrgen_encode_{{ type }}(xdr, &value->{{ name }}.element[i])) +{% else %} + if (!xdrgen_encode_{{ type }}(xdr, value->{{ name }}.element[i])) +{% endif %} + return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/variable_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/variable_length_opaque.j2 new file mode 100644 index 000000000000..1d477c2d197a --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/variable_length_opaque.j2 @@ -0,0 +1,8 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (variable-length opaque) */ +{% endif %} + if (value->{{ name }}.len > {{ maxsize }}) + return false; + if (xdr_stream_encode_opaque(xdr, value->{{ name }}.data, value->{{ name }}.len) < 0) + return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/variable_length_string.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/variable_length_string.j2 new file mode 100644 index 000000000000..cf65b71eaef3 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/variable_length_string.j2 @@ -0,0 +1,8 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (variable-length string) */ +{% endif %} + if (value->{{ name }}.len > {{ maxsize }}) + return false; + if (xdr_stream_encode_opaque(xdr, value->{{ name }}.data, value->{{ name }}.len) < 0) + return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/declaration/basic.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/declaration/basic.j2 new file mode 100644 index 000000000000..455b10bd90ec --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/declaration/basic.j2 @@ -0,0 +1,8 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +bool xdrgen_decode_{{ name }}(struct xdr_stream *xdr, {{ name }} *ptr); +{% if name in pass_by_reference %} +bool xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const {{ name }} *value); +{%- else -%} +bool xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const {{ name }} value); +{% endif %} diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/declaration/fixed_length_array.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/declaration/fixed_length_array.j2 new file mode 100644 index 000000000000..3fe3ddd9f359 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/declaration/fixed_length_array.j2 @@ -0,0 +1,4 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +bool xdrgen_decode_{{ name }}(struct xdr_stream *xdr, {{ classifier }}{{ name }} *ptr); +bool xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const {{ classifier }}{{ name }} value); diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/declaration/fixed_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/declaration/fixed_length_opaque.j2 new file mode 100644 index 000000000000..3fe3ddd9f359 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/declaration/fixed_length_opaque.j2 @@ -0,0 +1,4 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +bool xdrgen_decode_{{ name }}(struct xdr_stream *xdr, {{ classifier }}{{ name }} *ptr); +bool xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const {{ classifier }}{{ name }} value); diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/declaration/variable_length_array.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/declaration/variable_length_array.j2 new file mode 100644 index 000000000000..3fe3ddd9f359 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/declaration/variable_length_array.j2 @@ -0,0 +1,4 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +bool xdrgen_decode_{{ name }}(struct xdr_stream *xdr, {{ classifier }}{{ name }} *ptr); +bool xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const {{ classifier }}{{ name }} value); diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/declaration/variable_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/declaration/variable_length_opaque.j2 new file mode 100644 index 000000000000..3fe3ddd9f359 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/declaration/variable_length_opaque.j2 @@ -0,0 +1,4 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +bool xdrgen_decode_{{ name }}(struct xdr_stream *xdr, {{ classifier }}{{ name }} *ptr); +bool xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const {{ classifier }}{{ name }} value); diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/declaration/variable_length_string.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/declaration/variable_length_string.j2 new file mode 100644 index 000000000000..3fe3ddd9f359 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/declaration/variable_length_string.j2 @@ -0,0 +1,4 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +bool xdrgen_decode_{{ name }}(struct xdr_stream *xdr, {{ classifier }}{{ name }} *ptr); +bool xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const {{ classifier }}{{ name }} value); diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/basic.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/basic.j2 new file mode 100644 index 000000000000..da4709403dc9 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/basic.j2 @@ -0,0 +1,17 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* typedef {{ name }} */ +{% endif %} +{% if name in public_apis %} +bool +{% else %} +static bool __maybe_unused +{% endif %} +xdrgen_decode_{{ name }}(struct xdr_stream *xdr, {{ name }} *ptr) +{ +{% if annotate %} + /* (basic) */ +{% endif %} + return xdrgen_decode_{{ type }}(xdr, ptr); +}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/fixed_length_array.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/fixed_length_array.j2 new file mode 100644 index 000000000000..d7c80e472fe3 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/fixed_length_array.j2 @@ -0,0 +1,25 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* typedef {{ name }} */ +{% endif %} +{% if name in public_apis %} +bool +{% else %} +static bool __maybe_unused +{% endif %} +xdrgen_decode_{{ name }}(struct xdr_stream *xdr, {{ classifier }}{{ name }} *ptr) +{ +{% if annotate %} + /* (fixed-length array) */ +{% endif %} + for (u32 i = 0; i < {{ size }}; i++) { +{%- if classifier == '' %} + if (xdrgen_decode_{{ type }}(xdr, ptr->items[i]) < 0) +{% else %} + if (xdrgen_decode_{{ type }}(xdr, &ptr->items[i]) < 0) +{% endif %} + return false; + } + return true; +}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/fixed_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/fixed_length_opaque.j2 new file mode 100644 index 000000000000..8b4ff08c49e5 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/fixed_length_opaque.j2 @@ -0,0 +1,17 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* typedef {{ name }} */ +{% endif %} +{% if name in public_apis %} +bool +{% else %} +static bool __maybe_unused +{% endif %} +xdrgen_decode_{{ name }}(struct xdr_stream *xdr, {{ classifier }}{{ name }} *ptr) +{ +{% if annotate %} + /* (fixed-length opaque) */ +{% endif %} + return xdr_stream_decode_opaque_fixed(xdr, ptr, {{ size }}) >= 0; +}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/variable_length_array.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/variable_length_array.j2 new file mode 100644 index 000000000000..e74ffdd98463 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/variable_length_array.j2 @@ -0,0 +1,26 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* typedef {{ name }} */ +{% endif %} +{% if name in public_apis %} +bool +{% else %} +static bool __maybe_unused +{% endif %} +xdrgen_decode_{{ name }}(struct xdr_stream *xdr, {{ classifier }}{{ name }} *ptr) +{ +{% if annotate %} + /* (variable-length array) */ +{% endif %} + if (xdr_stream_decode_u32(xdr, &ptr->count) < 0) + return false; +{% if maxsize != "0" %} + if (ptr->count > {{ maxsize }}) + return false; +{% endif %} + for (u32 i = 0; i < ptr->count; i++) + if (!xdrgen_decode_{{ type }}(xdr, &ptr->element[i])) + return false; + return true; +}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/variable_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/variable_length_opaque.j2 new file mode 100644 index 000000000000..c1b7ad84f99c --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/variable_length_opaque.j2 @@ -0,0 +1,17 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* typedef {{ name }} */ +{% endif %} +{% if name in public_apis %} +bool +{% else %} +static bool __maybe_unused +{% endif %} +xdrgen_decode_{{ name }}(struct xdr_stream *xdr, {{ classifier }}{{ name }} *ptr) +{ +{% if annotate %} + /* (variable-length opaque) */ +{% endif %} + return xdr_stream_decode_opaque(xdr, ptr->data, ptr->len) >= 0; +}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/variable_length_string.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/variable_length_string.j2 new file mode 100644 index 000000000000..937286d76688 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/variable_length_string.j2 @@ -0,0 +1,17 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* typedef {{ name }} */ +{% endif %} +{% if name in public_apis %} +bool +{% else %} +static bool __maybe_unused +{% endif %} +xdrgen_decode_{{ name }}(struct xdr_stream *xdr, {{ classifier }}{{ name }} *ptr) +{ +{% if annotate %} + /* (variable-length string) */ +{% endif %} + return xdr_stream_decode_opaque(xdr, ptr->data, ptr->len) >= 0; +}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/definition/basic.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/definition/basic.j2 new file mode 100644 index 000000000000..1c5f28135eec --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/definition/basic.j2 @@ -0,0 +1,6 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* typedef {{ name }} (basic) */ +{% endif %} +typedef {{ classifier }}{{ type }} {{ name }}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/definition/fixed_length_array.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/definition/fixed_length_array.j2 new file mode 100644 index 000000000000..c3a67c952e77 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/definition/fixed_length_array.j2 @@ -0,0 +1,6 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* typedef {{ name }} (fixed-length array) */ +{% endif %} +typedef {{ type }}{{ name }}[{{ size }}]; diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/definition/fixed_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/definition/fixed_length_opaque.j2 new file mode 100644 index 000000000000..8788b02fe4f5 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/definition/fixed_length_opaque.j2 @@ -0,0 +1,6 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* typedef {{ name }} (fixed-length opaque) */ +{% endif %} +typedef u8 {{ name }}[{{ size }}]; diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/definition/variable_length_array.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/definition/variable_length_array.j2 new file mode 100644 index 000000000000..f03393760545 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/definition/variable_length_array.j2 @@ -0,0 +1,9 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* typedef {{ name }} (variable-length array) */ +{% endif %} +typedef struct { + u32 count; + {{ classifier }}{{ type }} *element; +} {{ name }}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/definition/variable_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/definition/variable_length_opaque.j2 new file mode 100644 index 000000000000..162f2610af34 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/definition/variable_length_opaque.j2 @@ -0,0 +1,6 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* typedef {{ name }} (variable-length opaque) */ +{% endif %} +typedef opaque {{ name }}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/definition/variable_length_string.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/definition/variable_length_string.j2 new file mode 100644 index 000000000000..c03c2df8e625 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/definition/variable_length_string.j2 @@ -0,0 +1,6 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* typedef {{ name }} (variable-length string) */ +{% endif %} +typedef string {{ name }}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/basic.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/basic.j2 new file mode 100644 index 000000000000..35effe67e4ef --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/basic.j2 @@ -0,0 +1,21 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* typedef {{ name }} */ +{% endif %} +{% if name in public_apis %} +bool +{% else %} +static bool __maybe_unused +{% endif %} +{% if name in pass_by_reference %} +xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const {{ classifier }}{{ name }} *value) +{% else %} +xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const {{ classifier }}{{ name }} value) +{% endif %} +{ +{% if annotate %} + /* (basic) */ +{% endif %} + return xdrgen_encode_{{ type }}(xdr, value); +}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/fixed_length_array.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/fixed_length_array.j2 new file mode 100644 index 000000000000..95202ad5ad2d --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/fixed_length_array.j2 @@ -0,0 +1,25 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* typedef {{ name }} */ +{% endif %} +{% if name in public_apis %} +bool +{% else %} +static bool __maybe_unused +{% endif %} +xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const {{ classifier }}{{ name }} value) +{ +{% if annotate %} + /* (fixed-length array) */ +{% endif %} + for (u32 i = 0; i < {{ size }}; i++) { +{% if type in pass_by_reference %} + if (xdrgen_encode_{{ type }}(xdr, &value->items[i]) < 0) +{% else %} + if (xdrgen_encode_{{ type }}(xdr, value->items[i]) < 0) +{% endif %} + return false; + } + return true; +}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/fixed_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/fixed_length_opaque.j2 new file mode 100644 index 000000000000..9c66a11b9912 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/fixed_length_opaque.j2 @@ -0,0 +1,17 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* typedef {{ name }} */ +{% endif %} +{% if name in public_apis %} +bool +{% else %} +static bool __maybe_unused +{% endif %} +xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const {{ classifier }}{{ name }} value) +{ +{% if annotate %} + /* (fixed-length opaque) */ +{% endif %} + return xdr_stream_encode_opaque_fixed(xdr, value, {{ size }}) >= 0; +}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/variable_length_array.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/variable_length_array.j2 new file mode 100644 index 000000000000..2d2384f64918 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/variable_length_array.j2 @@ -0,0 +1,30 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* typedef {{ name }} */ +{% endif %} +{% if name in public_apis %} +bool +{% else %} +static bool __maybe_unused +{% endif %} +xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const {{ classifier }}{{ name }} value) +{ +{% if annotate %} + /* (variable-length array) */ +{% endif %} +{% if maxsize != "0" %} + if (unlikely(value.count > {{ maxsize }})) + return false; +{% endif %} + if (xdr_stream_encode_u32(xdr, value.count) != XDR_UNIT) + return false; + for (u32 i = 0; i < value.count; i++) +{% if type in pass_by_reference %} + if (!xdrgen_encode_{{ type }}(xdr, &value.element[i])) +{% else %} + if (!xdrgen_encode_{{ type }}(xdr, value.element[i])) +{% endif %} + return false; + return true; +}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/variable_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/variable_length_opaque.j2 new file mode 100644 index 000000000000..8508f13c95b9 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/variable_length_opaque.j2 @@ -0,0 +1,17 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* typedef {{ name }} */ +{% endif %} +{% if name in public_apis %} +bool +{% else %} +static bool __maybe_unused +{% endif %} +xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const {{ classifier }}{{ name }} value) +{ +{% if annotate %} + /* (variable-length opaque) */ +{% endif %} + return xdr_stream_encode_opaque(xdr, value.data, value.len) >= 0; +}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/variable_length_string.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/variable_length_string.j2 new file mode 100644 index 000000000000..3d490ff180d0 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/variable_length_string.j2 @@ -0,0 +1,17 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* typedef {{ name }} */ +{% endif %} +{% if name in public_apis %} +bool +{% else %} +static bool __maybe_unused +{% endif %} +xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const {{ classifier }}{{ name }} value) +{ +{% if annotate %} + /* (variable-length string) */ +{% endif %} + return xdr_stream_encode_opaque(xdr, value.data, value.len) >= 0; +}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/decoder/basic.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/basic.j2 new file mode 100644 index 000000000000..4d97cc5395eb --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/basic.j2 @@ -0,0 +1,6 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (basic) */ +{% endif %} + if (!xdrgen_decode_{{ type }}(xdr, &ptr->u.{{ name }})) + return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/decoder/break.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/break.j2 new file mode 100644 index 000000000000..b286d1407029 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/break.j2 @@ -0,0 +1,2 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + break; diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/decoder/case_spec.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/case_spec.j2 new file mode 100644 index 000000000000..5fa2163f0a74 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/case_spec.j2 @@ -0,0 +1,2 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + case {{ case }}: diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/decoder/close.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/close.j2 new file mode 100644 index 000000000000..fdc2dfd1843b --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/close.j2 @@ -0,0 +1,4 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + } + return true; +}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/decoder/default_spec.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/default_spec.j2 new file mode 100644 index 000000000000..044a002d0589 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/default_spec.j2 @@ -0,0 +1,2 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + default: diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/decoder/open.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/open.j2 new file mode 100644 index 000000000000..eb9941376e49 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/open.j2 @@ -0,0 +1,12 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* union {{ name }} */ +{% endif %} +{% if name in public_apis %} +bool +{% else %} +static bool __maybe_unused +{% endif %} +xdrgen_decode_{{ name }}(struct xdr_stream *xdr, struct {{ name }} *ptr) +{ diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/decoder/optional_data.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/optional_data.j2 new file mode 100644 index 000000000000..e4476f5fd8d3 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/optional_data.j2 @@ -0,0 +1,6 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (optional data) */ +{% endif %} + if (!xdrgen_decode_{{ type }}(xdr, &ptr->u.{{ name }})) + return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/decoder/switch_spec.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/switch_spec.j2 new file mode 100644 index 000000000000..99b3067ef617 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/switch_spec.j2 @@ -0,0 +1,7 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* discriminant {{ name }} */ +{% endif %} + if (!xdrgen_decode_{{ type }}(xdr, &ptr->{{ name }})) + return false; + switch (ptr->{{ name }}) { diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/decoder/variable_length_array.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/variable_length_array.j2 new file mode 100644 index 000000000000..eee2b9a68e27 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/variable_length_array.j2 @@ -0,0 +1,13 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (variable-length array) */ +{% endif %} + if (xdr_stream_decode_u32(xdr, &count) != XDR_UNIT) + return false; + if (count > {{ maxsize }}) + return false; + for (u32 i = 0; i < count; i++) { + if (xdrgen_decode_{{ type }}(xdr, &ptr->{{ name }}.items[i]) < 0) + return false; + } + ptr->{{ name }}.len = count; diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/decoder/variable_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/variable_length_opaque.j2 new file mode 100644 index 000000000000..c9d88ed29c78 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/variable_length_opaque.j2 @@ -0,0 +1,6 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (variable-length opaque) */ +{% endif %} + if (!xdrgen_decode_opaque(xdr, (struct opaque *)ptr->u.{{ name }}, {{ maxsize }})) + return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/decoder/variable_length_string.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/variable_length_string.j2 new file mode 100644 index 000000000000..83b6e5a14e7f --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/variable_length_string.j2 @@ -0,0 +1,6 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (variable-length string) */ +{% endif %} + if (!xdrgen_decode_string(xdr, (struct string *)ptr->u.{{ name }}, {{ maxsize }})) + return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/decoder/void.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/void.j2 new file mode 100644 index 000000000000..65205ce37b36 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/void.j2 @@ -0,0 +1,3 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + if (!xdrgen_decode_void(xdr)) + return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/definition/case_spec.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/definition/case_spec.j2 new file mode 100644 index 000000000000..52f8d131b805 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/union/definition/case_spec.j2 @@ -0,0 +1,2 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + {{ classifier }}{{ type }} {{ name }}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/definition/close.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/definition/close.j2 new file mode 100644 index 000000000000..01d716d0099e --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/union/definition/close.j2 @@ -0,0 +1,8 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + } u; +}; +{%- if name in public_apis %} + +bool xdrgen_decode_{{ name }}(struct xdr_stream *xdr, struct {{ name }} *ptr); +bool xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const struct {{ name }} *ptr); +{%- endif -%} diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/definition/default_spec.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/definition/default_spec.j2 new file mode 100644 index 000000000000..52f8d131b805 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/union/definition/default_spec.j2 @@ -0,0 +1,2 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + {{ classifier }}{{ type }} {{ name }}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/definition/open.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/definition/open.j2 new file mode 100644 index 000000000000..20fcfd1fc4e5 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/union/definition/open.j2 @@ -0,0 +1,6 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* union {{ name }} */ +{% endif %} +struct {{ name }} { diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/definition/switch_spec.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/definition/switch_spec.j2 new file mode 100644 index 000000000000..3e552732502c --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/union/definition/switch_spec.j2 @@ -0,0 +1,3 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + {{ classifier }}{{ type }} {{ name }}; + union { diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/encoder/basic.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/encoder/basic.j2 new file mode 100644 index 000000000000..6452d75c6f9a --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/union/encoder/basic.j2 @@ -0,0 +1,10 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (basic) */ +{% endif %} +{% if type in pass_by_reference %} + if (!xdrgen_encode_{{ type }}(xdr, &ptr->u.{{ name }})) +{% else %} + if (!xdrgen_encode_{{ type }}(xdr, ptr->u.{{ name }})) +{% endif %} + return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/encoder/break.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/encoder/break.j2 new file mode 100644 index 000000000000..b286d1407029 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/union/encoder/break.j2 @@ -0,0 +1,2 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + break; diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/encoder/case_spec.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/encoder/case_spec.j2 new file mode 100644 index 000000000000..5fa2163f0a74 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/union/encoder/case_spec.j2 @@ -0,0 +1,2 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + case {{ case }}: diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/encoder/close.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/encoder/close.j2 new file mode 100644 index 000000000000..fdc2dfd1843b --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/union/encoder/close.j2 @@ -0,0 +1,4 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + } + return true; +}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/encoder/default_spec.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/encoder/default_spec.j2 new file mode 100644 index 000000000000..044a002d0589 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/union/encoder/default_spec.j2 @@ -0,0 +1,2 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + default: diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/encoder/open.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/encoder/open.j2 new file mode 100644 index 000000000000..e5a206df10c6 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/union/encoder/open.j2 @@ -0,0 +1,12 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* union {{ name }} */ +{% endif %} +{% if name in public_apis %} +bool +{% else %} +static bool __maybe_unused +{% endif %} +xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const struct {{ name }} *ptr) +{ diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/encoder/switch_spec.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/encoder/switch_spec.j2 new file mode 100644 index 000000000000..c8c3ecbe038b --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/union/encoder/switch_spec.j2 @@ -0,0 +1,7 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* discriminant {{ name }} */ +{% endif %} + if (!xdrgen_encode_{{ type }}(xdr, ptr->{{ name }})) + return false; + switch (ptr->{{ name }}) { diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/encoder/void.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/encoder/void.j2 new file mode 100644 index 000000000000..84e7c2127d75 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/union/encoder/void.j2 @@ -0,0 +1,3 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + if (!xdrgen_encode_void(xdr)) + return false; diff --git a/tools/net/sunrpc/xdrgen/tests/test.x b/tools/net/sunrpc/xdrgen/tests/test.x new file mode 100644 index 000000000000..90c8587f6fe5 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/tests/test.x @@ -0,0 +1,36 @@ +/* Sample XDR specification from RFC 1832 Section 5.5 */ + +const MAXUSERNAME = 32; /* max length of a user name */ +const MAXFILELEN = 65535; /* max length of a file */ +const MAXNAMELEN = 255; /* max length of a file name */ + +/* + * Types of files: + */ +enum filekind { + TEXT = 0, /* ascii data */ + DATA = 1, /* raw data */ + EXEC = 2 /* executable */ +}; + +/* + * File information, per kind of file: + */ +union filetype switch (filekind kind) { +case TEXT: + void; /* no extra information */ +case DATA: + string creator; /* data creator */ +case EXEC: + string interpretor; /* program interpretor */ +}; + +/* + * A complete file: + */ +struct file { + string filename; /* name of file */ + filetype type; /* info about file */ + string owner; /* owner of file */ + opaque data; /* file data */ +}; diff --git a/tools/net/sunrpc/xdrgen/xdr_ast.py b/tools/net/sunrpc/xdrgen/xdr_ast.py new file mode 100644 index 000000000000..dbd3fcf9c957 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/xdr_ast.py @@ -0,0 +1,510 @@ +#!/usr/bin/env python3 +# ex: set filetype=python: + +"""Define and implement the Abstract Syntax Tree for the XDR language.""" + +import sys +from typing import List +from dataclasses import dataclass + +from lark import ast_utils, Transformer +from lark.tree import Meta + +this_module = sys.modules[__name__] + +excluded_apis = [] +header_name = "none" +public_apis = [] +enums = set() +structs = set() +pass_by_reference = set() + + +@dataclass +class _XdrAst(ast_utils.Ast): + """Base class for the XDR abstract syntax tree""" + + +@dataclass +class _XdrIdentifier(_XdrAst): + """Corresponds to 'identifier' in the XDR language grammar""" + + symbol: str + + +@dataclass +class _XdrValue(_XdrAst): + """Corresponds to 'value' in the XDR language grammar""" + + value: str + + +@dataclass +class _XdrConstantValue(_XdrAst): + """Corresponds to 'constant' in the XDR language grammar""" + + value: int + + +@dataclass +class _XdrTypeSpecifier(_XdrAst): + """Corresponds to 'type_specifier' in the XDR language grammar""" + + type_name: str + c_classifier: str + + +@dataclass +class _XdrDefinedType(_XdrTypeSpecifier): + """Corresponds to a type defined by the input specification""" + + +@dataclass +class _XdrBuiltInType(_XdrTypeSpecifier): + """Corresponds to a built-in XDR type""" + + +@dataclass +class _XdrDeclaration(_XdrAst): + """Base class of XDR type declarations""" + + +@dataclass +class _XdrFixedLengthOpaque(_XdrDeclaration): + """A fixed-length opaque declaration""" + + name: str + size: str + template: str = "fixed_length_opaque" + + +@dataclass +class _XdrVariableLengthOpaque(_XdrDeclaration): + """A variable-length opaque declaration""" + + name: str + maxsize: str + template: str = "variable_length_opaque" + + +@dataclass +class _XdrVariableLengthString(_XdrDeclaration): + """A (NUL-terminated) variable-length string declaration""" + + name: str + maxsize: str + template: str = "variable_length_string" + + +@dataclass +class _XdrFixedLengthArray(_XdrDeclaration): + """A fixed-length array declaration""" + + name: str + spec: _XdrTypeSpecifier + size: str + template: str = "fixed_length_array" + + +@dataclass +class _XdrVariableLengthArray(_XdrDeclaration): + """A variable-length array declaration""" + + name: str + spec: _XdrTypeSpecifier + maxsize: str + template: str = "variable_length_array" + + +@dataclass +class _XdrOptionalData(_XdrDeclaration): + """An 'optional_data' declaration""" + + name: str + spec: _XdrTypeSpecifier + template: str = "optional_data" + + +@dataclass +class _XdrBasic(_XdrDeclaration): + """A 'basic' declaration""" + + name: str + spec: _XdrTypeSpecifier + template: str = "basic" + + +@dataclass +class _XdrVoid(_XdrDeclaration): + """A void declaration""" + + template: str = "void" + + +@dataclass +class _XdrConstant(_XdrAst): + """Corresponds to 'constant_def' in the grammar""" + + name: str + value: str + + +@dataclass +class _XdrEnumerator(_XdrAst): + """An 'identifier = value' enumerator""" + + name: str + value: str + + +@dataclass +class _XdrEnum(_XdrAst): + """An XDR enum definition""" + + name: str + minimum: int + maximum: int + enumerators: List[_XdrEnumerator] + + +@dataclass +class _XdrStruct(_XdrAst): + """An XDR struct definition""" + + name: str + fields: List[_XdrDeclaration] + + +@dataclass +class _XdrPointer(_XdrAst): + """An XDR pointer definition""" + + name: str + fields: List[_XdrDeclaration] + + +@dataclass +class _XdrTypedef(_XdrAst): + """An XDR typedef""" + + declaration: _XdrDeclaration + + +@dataclass +class _XdrCaseSpec(_XdrAst): + """One case in an XDR union""" + + values: List[str] + arm: _XdrDeclaration + template: str = "case_spec" + + +@dataclass +class _XdrDefaultSpec(_XdrAst): + """Default case in an XDR union""" + + arm: _XdrDeclaration + template: str = "default_spec" + + +@dataclass +class _XdrUnion(_XdrAst): + """An XDR union""" + + name: str + discriminant: _XdrDeclaration + cases: List[_XdrCaseSpec] + default: _XdrDeclaration + + +@dataclass +class _RpcProcedure(_XdrAst): + """RPC procedure definition""" + + name: str + number: str + argument: _XdrTypeSpecifier + result: _XdrTypeSpecifier + + +@dataclass +class _RpcVersion(_XdrAst): + """RPC version definition""" + + name: str + number: str + procedures: List[_RpcProcedure] + + +@dataclass +class _RpcProgram(_XdrAst): + """RPC program definition""" + + name: str + number: str + versions: List[_RpcVersion] + + +@dataclass +class _Pragma(_XdrAst): + """Empty class for pragma directives""" + + +@dataclass +class Definition(_XdrAst, ast_utils.WithMeta): + """Corresponds to 'definition' in the grammar""" + + meta: Meta + value: _XdrAst + + +@dataclass +class Specification(_XdrAst, ast_utils.AsList): + """Corresponds to 'specification' in the grammar""" + + definitions: List[Definition] + + +class ParseToAst(Transformer): + """Functions that transform productions into AST nodes""" + + def identifier(self, children): + """Instantiate one _XdrIdentifier object""" + return _XdrIdentifier(children[0].value) + + def value(self, children): + """Instantiate one _XdrValue object""" + if isinstance(children[0], _XdrIdentifier): + return _XdrValue(children[0].symbol) + return _XdrValue(children[0].children[0].value) + + def constant(self, children): + """Instantiate one _XdrConstantValue object""" + match children[0].data: + case "decimal_constant": + value = int(children[0].children[0].value, base=10) + case "hexadecimal_constant": + value = int(children[0].children[0].value, base=16) + case "octal_constant": + value = int(children[0].children[0].value, base=8) + return _XdrConstantValue(value) + + def type_specifier(self, children): + """Instantiate one type_specifier object""" + c_classifier = "" + if isinstance(children[0], _XdrIdentifier): + name = children[0].symbol + if name in enums: + c_classifier = "enum " + if name in structs: + c_classifier = "struct " + return _XdrDefinedType( + type_name=name, + c_classifier=c_classifier, + ) + + token = children[0].data + return _XdrBuiltInType( + type_name=token.value, + c_classifier=c_classifier, + ) + + def constant_def(self, children): + """Instantiate one _XdrConstant object""" + name = children[0].symbol + value = children[1].value + return _XdrConstant(name, value) + + # cel: Python can compute a min() and max() for the enumerator values + # so that the generated code can perform proper range checking. + def enum(self, children): + """Instantiate one _XdrEnum object""" + enum_name = children[0].symbol + enums.add(enum_name) + + i = 0 + enumerators = [] + body = children[1] + while i < len(body.children): + name = body.children[i].symbol + value = body.children[i + 1].value + enumerators.append(_XdrEnumerator(name, value)) + i = i + 2 + + return _XdrEnum(enum_name, 0, 0, enumerators) + + def fixed_length_opaque(self, children): + """Instantiate one _XdrFixedLengthOpaque declaration object""" + name = children[0].symbol + size = children[1].value + + return _XdrFixedLengthOpaque(name, size) + + def variable_length_opaque(self, children): + """Instantiate one _XdrVariableLengthOpaque declaration object""" + name = children[0].symbol + if children[1] is not None: + maxsize = children[1].value + else: + maxsize = "0" + + return _XdrVariableLengthOpaque(name, maxsize) + + def variable_length_string(self, children): + """Instantiate one _XdrVariableLengthString declaration object""" + name = children[0].symbol + if children[1] is not None: + maxsize = children[1].value + else: + maxsize = "0" + + return _XdrVariableLengthString(name, maxsize) + + def fixed_length_array(self, children): + """Instantiate one _XdrFixedLengthArray declaration object""" + spec = children[0] + name = children[1].symbol + size = children[2].value + + return _XdrFixedLengthArray(name, spec, size) + + def variable_length_array(self, children): + """Instantiate one _XdrVariableLengthArray declaration object""" + spec = children[0] + name = children[1].symbol + if children[2] is not None: + maxsize = children[2].value + else: + maxsize = "0" + + return _XdrVariableLengthArray(name, spec, maxsize) + + def optional_data(self, children): + """Instantiate one _XdrOptionalData declaration object""" + spec = children[0] + name = children[1].symbol + structs.add(name) + pass_by_reference.add(name) + + return _XdrOptionalData(name, spec) + + def basic(self, children): + """Instantiate one _XdrBasic object""" + spec = children[0] + name = children[1].symbol + + return _XdrBasic(name, spec) + + def void(self, children): + """Instantiate one _XdrVoid declaration object""" + + return _XdrVoid() + + def struct(self, children): + """Instantiate one _XdrStruct object""" + name = children[0].symbol + structs.add(name) + pass_by_reference.add(name) + fields = children[1].children + + last_field = fields[-1] + if ( + isinstance(last_field, _XdrOptionalData) + and name == last_field.spec.type_name + ): + return _XdrPointer(name, fields) + + return _XdrStruct(name, fields) + + def typedef(self, children): + """Instantiate one _XdrTypedef object""" + new_type = children[0] + if isinstance(new_type, _XdrBasic) and isinstance( + new_type.spec, _XdrDefinedType + ): + if new_type.spec.type_name in pass_by_reference: + pass_by_reference.add(new_type.name) + + return _XdrTypedef(new_type) + + def case_spec(self, children): + """Instantiate one _XdrCaseSpec object""" + values = [] + for item in children[0:-1]: + values.append(item.value) + arm = children[-1] + + return _XdrCaseSpec(values, arm) + + def default_spec(self, children): + """Instantiate one _XdrDefaultSpec object""" + arm = children[0] + + return _XdrDefaultSpec(arm) + + def union(self, children): + """Instantiate one _XdrUnion object""" + name = children[0].symbol + structs.add(name) + pass_by_reference.add(name) + + body = children[1] + discriminant = body.children[0].children[0] + cases = body.children[1:-1] + default = body.children[-1] + + return _XdrUnion(name, discriminant, cases, default) + + def procedure_def(self, children): + """Instantiate one _RpcProcedure object""" + result = children[0] + name = children[1].symbol + argument = children[2] + number = children[3].value + + return _RpcProcedure(name, number, argument, result) + + def version_def(self, children): + """Instantiate one _RpcVersion object""" + name = children[0].symbol + number = children[-1].value + procedures = children[1:-1] + + return _RpcVersion(name, number, procedures) + + def program_def(self, children): + """Instantiate one _RpcProgram object""" + name = children[0].symbol + number = children[-1].value + versions = children[1:-1] + + return _RpcProgram(name, number, versions) + + def pragma_def(self, children): + """Instantiate one _Pragma object""" + directive = children[0].children[0].data + match directive: + case "exclude_directive": + excluded_apis.append(children[1].symbol) + case "header_directive": + global header_name + header_name = children[1].symbol + case "public_directive": + public_apis.append(children[1].symbol) + case _: + raise NotImplementedError("Directive not supported") + return _Pragma() + + +transformer = ast_utils.create_transformer(this_module, ParseToAst()) + + +def transform_parse_tree(parse_tree): + """Transform productions into an abstract syntax tree""" + + return transformer.transform(parse_tree) + + +def get_header_name() -> str: + """Return header name set by pragma header directive""" + return header_name diff --git a/tools/net/sunrpc/xdrgen/xdr_parse.py b/tools/net/sunrpc/xdrgen/xdr_parse.py new file mode 100644 index 000000000000..964b44e675df --- /dev/null +++ b/tools/net/sunrpc/xdrgen/xdr_parse.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +# ex: set filetype=python: + +"""Common parsing code for xdrgen""" + +from lark import Lark + + +# Set to True to emit annotation comments in generated source +annotate = False + + +def set_xdr_annotate(set_it: bool) -> None: + """Set 'annotate' if --annotate was specified on the command line""" + global annotate + annotate = set_it + + +def get_xdr_annotate() -> bool: + """Return True if --annotate was specified on the command line""" + return annotate + + +def xdr_parser() -> Lark: + """Return a Lark parser instance configured with the XDR language grammar""" + + return Lark.open( + "grammars/xdr.lark", + rel_to=__file__, + start="specification", + debug=True, + strict=True, + propagate_positions=True, + parser="lalr", + lexer="contextual", + ) diff --git a/tools/net/sunrpc/xdrgen/xdrgen b/tools/net/sunrpc/xdrgen/xdrgen new file mode 100755 index 000000000000..95f303b2861b --- /dev/null +++ b/tools/net/sunrpc/xdrgen/xdrgen @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +# ex: set filetype=python: + +"""Translate an XDR specification into executable code that +can be compiled for the Linux kernel.""" + +__author__ = "Chuck Lever" +__copyright__ = "Copyright (c) 2024 Oracle and/or its affiliates." +__license__ = "GPL-2.0 only" +__version__ = "0.2" + +import sys +import argparse + +from subcmds import definitions +from subcmds import declarations +from subcmds import lint +from subcmds import source + + +sys.path.insert(1, "@pythondir@") + + +def main() -> int: + """Parse command-line options""" + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description="Convert an XDR specification to Linux kernel source code", + epilog="""\ +Copyright (c) 2024 Oracle and/or its affiliates. + +License GPLv2: +This is free software. You are free to change and redistribute it. +There is NO WARRANTY, to the extent permitted by law.""", + ) + parser.add_argument( + "--version", + help="Display the version of this tool", + action="version", + version=__version__, + ) + + subcommands = parser.add_subparsers(title="Subcommands", required=True) + + definitions_parser = subcommands.add_parser( + "definitions", help="Generate XDR definitions" + ) + definitions_parser.add_argument( + "--annotate", + action="store_true", + default=False, + help="Add annotation comments", + ) + definitions_parser.add_argument( + "--language", + action="store_true", + default="C", + help="Output language", + ) + definitions_parser.add_argument( + "--peer", + choices=["server", "client",], + default="server", + help="Generate header code for client or server side", + type=str, + ) + definitions_parser.add_argument("filename", help="File containing an XDR specification") + definitions_parser.set_defaults(func=definitions.subcmd) + + declarations_parser = subcommands.add_parser( + "declarations", help="Generate function declarations" + ) + declarations_parser.add_argument( + "--annotate", + action="store_true", + default=False, + help="Add annotation comments", + ) + declarations_parser.add_argument( + "--language", + action="store_true", + default="C", + help="Output language", + ) + declarations_parser.add_argument( + "--peer", + choices=["server", "client",], + default="server", + help="Generate code for client or server side", + type=str, + ) + declarations_parser.add_argument("filename", help="File containing an XDR specification") + declarations_parser.set_defaults(func=declarations.subcmd) + + linter_parser = subcommands.add_parser("lint", help="Check an XDR specification") + linter_parser.add_argument("filename", help="File containing an XDR specification") + linter_parser.set_defaults(func=lint.subcmd) + + source_parser = subcommands.add_parser( + "source", help="Generate XDR encoder and decoder source code" + ) + source_parser.add_argument( + "--annotate", + action="store_true", + default=False, + help="Add annotation comments", + ) + source_parser.add_argument( + "--language", + action="store_true", + default="C", + help="Output language", + ) + source_parser.add_argument( + "--peer", + choices=["server", "client",], + default="server", + help="Generate code for client or server side", + type=str, + ) + source_parser.add_argument("filename", help="File containing an XDR specification") + source_parser.set_defaults(func=source.subcmd) + + args = parser.parse_args() + return args.func(args) + + +try: + if __name__ == "__main__": + sys.exit(main()) +except (SystemExit, KeyboardInterrupt, BrokenPipeError): + sys.exit(1) From 663ad8b1df8724cd5e01df66ea67ce0424fbcdf6 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 10 Sep 2024 15:31:19 -0400 Subject: [PATCH 262/302] xdrgen: Fix return code checking in built-in XDR decoders xdr_stream_encode_u32() returns XDR_UNIT on success. xdr_stream_decode_u32() returns zero or -EMSGSIZE, but never XDR_UNIT. Signed-off-by: Chuck Lever --- include/linux/sunrpc/xdrgen/_builtins.h | 4 ++-- .../templates/C/pointer/decoder/variable_length_array.j2 | 2 +- .../templates/C/struct/decoder/variable_length_array.j2 | 2 +- .../xdrgen/templates/C/union/decoder/variable_length_array.j2 | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/linux/sunrpc/xdrgen/_builtins.h b/include/linux/sunrpc/xdrgen/_builtins.h index 68746c59fc9a..66ca3ece951a 100644 --- a/include/linux/sunrpc/xdrgen/_builtins.h +++ b/include/linux/sunrpc/xdrgen/_builtins.h @@ -184,7 +184,7 @@ xdrgen_decode_string(struct xdr_stream *xdr, string *ptr, u32 maxlen) __be32 *p; u32 len; - if (unlikely(xdr_stream_decode_u32(xdr, &len) != XDR_UNIT)) + if (unlikely(xdr_stream_decode_u32(xdr, &len) < 0)) return false; if (unlikely(maxlen && len > maxlen)) return false; @@ -215,7 +215,7 @@ xdrgen_decode_opaque(struct xdr_stream *xdr, opaque *ptr, u32 maxlen) __be32 *p; u32 len; - if (unlikely(xdr_stream_decode_u32(xdr, &len) != XDR_UNIT)) + if (unlikely(xdr_stream_decode_u32(xdr, &len) < 0)) return false; if (unlikely(maxlen && len > maxlen)) return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/variable_length_array.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/variable_length_array.j2 index f54ccd136762..2f943909cdf7 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/variable_length_array.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/variable_length_array.j2 @@ -2,7 +2,7 @@ {% if annotate %} /* member {{ name }} (variable-length array) */ {% endif %} - if (xdr_stream_decode_u32(xdr, &ptr->{{ name }}.count) != XDR_UNIT) + if (xdr_stream_decode_u32(xdr, &ptr->{{ name }}.count) < 0) return false; {% if maxsize != "0" %} if (ptr->{{ name }}.count > {{ maxsize }}) diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/variable_length_array.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/variable_length_array.j2 index f54ccd136762..2f943909cdf7 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/variable_length_array.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/variable_length_array.j2 @@ -2,7 +2,7 @@ {% if annotate %} /* member {{ name }} (variable-length array) */ {% endif %} - if (xdr_stream_decode_u32(xdr, &ptr->{{ name }}.count) != XDR_UNIT) + if (xdr_stream_decode_u32(xdr, &ptr->{{ name }}.count) < 0) return false; {% if maxsize != "0" %} if (ptr->{{ name }}.count > {{ maxsize }}) diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/decoder/variable_length_array.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/variable_length_array.j2 index eee2b9a68e27..51ad736d2530 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/union/decoder/variable_length_array.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/variable_length_array.j2 @@ -2,7 +2,7 @@ {% if annotate %} /* member {{ name }} (variable-length array) */ {% endif %} - if (xdr_stream_decode_u32(xdr, &count) != XDR_UNIT) + if (xdr_stream_decode_u32(xdr, &count) < 0) return false; if (count > {{ maxsize }}) return false; From fed8a17c61ffa2ba53dc749068b6f07ecf40e3bf Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 10 Sep 2024 15:51:46 -0400 Subject: [PATCH 263/302] xdrgen: typedefs should use the built-in string and opaque functions 'typedef opaque yada' should use xdrgen's built-in opaque encoder and decoder, to enable better compiler optimization. Signed-off-by: Chuck Lever --- .../templates/C/typedef/decoder/variable_length_opaque.j2 | 2 +- .../templates/C/typedef/decoder/variable_length_string.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/variable_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/variable_length_opaque.j2 index c1b7ad84f99c..f28f8b228ad5 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/variable_length_opaque.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/variable_length_opaque.j2 @@ -13,5 +13,5 @@ xdrgen_decode_{{ name }}(struct xdr_stream *xdr, {{ classifier }}{{ name }} *ptr {% if annotate %} /* (variable-length opaque) */ {% endif %} - return xdr_stream_decode_opaque(xdr, ptr->data, ptr->len) >= 0; + return xdrgen_decode_opaque(xdr, ptr, {{ maxsize }}); }; diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/variable_length_string.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/variable_length_string.j2 index 937286d76688..56c5a17d6a70 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/variable_length_string.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/variable_length_string.j2 @@ -13,5 +13,5 @@ xdrgen_decode_{{ name }}(struct xdr_stream *xdr, {{ classifier }}{{ name }} *ptr {% if annotate %} /* (variable-length string) */ {% endif %} - return xdr_stream_decode_opaque(xdr, ptr->data, ptr->len) >= 0; + return xdrgen_decode_string(xdr, ptr, {{ maxsize }}); }; From 509abfc7a0ba66afa648e8216306acdc55ec54ed Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Fri, 13 Sep 2024 13:50:56 -0400 Subject: [PATCH 264/302] xdrgen: Prevent reordering of encoder and decoder functions I noticed that "xdrgen source" reorders the procedure encoder and decoder functions every time it is run. I would prefer that the generated code be more deterministic: it enables a reader to better see exactly what has changed between runs of the tool. The problem is that Python sets are not ordered. I use a Python set to ensure that, when multiple procedures use a particular argument or result type, the encoder/decoder for that type is emitted only once. Sets aren't ordered, but I can use Python dictionaries for this purpose to ensure the procedure functions are always emitted in the same order if the .x file does not change. Signed-off-by: Chuck Lever --- tools/net/sunrpc/xdrgen/generators/program.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tools/net/sunrpc/xdrgen/generators/program.py b/tools/net/sunrpc/xdrgen/generators/program.py index 83b0ecbae86f..ac3cf1694b68 100644 --- a/tools/net/sunrpc/xdrgen/generators/program.py +++ b/tools/net/sunrpc/xdrgen/generators/program.py @@ -34,20 +34,20 @@ def emit_version_declarations( environment: Environment, program: str, version: _RpcVersion ) -> None: """Emit declarations for each RPC version's procedures""" - arguments = set() + arguments = dict.fromkeys([]) for procedure in version.procedures: if procedure.name not in excluded_apis: - arguments.add(procedure.argument.type_name) + arguments[procedure.argument.type_name] = None if len(arguments) > 0: print("") template = environment.get_template("declaration/argument.j2") for argument in arguments: print(template.render(program=program, argument=argument)) - results = set() + results = dict.fromkeys([]) for procedure in version.procedures: if procedure.name not in excluded_apis: - results.add(procedure.result.type_name) + results[procedure.result.type_name] = None if len(results) > 0: print("") template = environment.get_template("declaration/result.j2") @@ -59,10 +59,10 @@ def emit_version_argument_decoders( environment: Environment, program: str, version: _RpcVersion ) -> None: """Emit server argument decoders for each RPC version's procedures""" - arguments = set() + arguments = dict.fromkeys([]) for procedure in version.procedures: if procedure.name not in excluded_apis: - arguments.add(procedure.argument.type_name) + arguments[procedure.argument.type_name] = None template = environment.get_template("decoder/argument.j2") for argument in arguments: @@ -73,10 +73,10 @@ def emit_version_result_decoders( environment: Environment, program: str, version: _RpcVersion ) -> None: """Emit client result decoders for each RPC version's procedures""" - results = set() + results = dict.fromkeys([]) for procedure in version.procedures: if procedure.name not in excluded_apis: - results.add(procedure.result.type_name) + results[procedure.result.type_name] = None template = environment.get_template("decoder/result.j2") for result in results: @@ -87,10 +87,10 @@ def emit_version_argument_encoders( environment: Environment, program: str, version: _RpcVersion ) -> None: """Emit client argument encoders for each RPC version's procedures""" - arguments = set() + arguments = dict.fromkeys([]) for procedure in version.procedures: if procedure.name not in excluded_apis: - arguments.add(procedure.argument.type_name) + arguments[procedure.argument.type_name] = None template = environment.get_template("encoder/argument.j2") for argument in arguments: @@ -101,10 +101,10 @@ def emit_version_result_encoders( environment: Environment, program: str, version: _RpcVersion ) -> None: """Emit server result encoders for each RPC version's procedures""" - results = set() + results = dict.fromkeys([]) for procedure in version.procedures: if procedure.name not in excluded_apis: - results.add(procedure.result.type_name) + results[procedure.result.type_name] = None template = environment.get_template("encoder/result.j2") for result in results: From 3340dee2350954c549b12c41206e98ed86d3b63e Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 1 Sep 2024 13:36:42 -0400 Subject: [PATCH 265/302] bcachefs: Add pinned to btree cache not freed counters Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_cache.c | 28 ++++++++++++++++------------ fs/bcachefs/btree_types.h | 29 ++++++++++++++++++++--------- 2 files changed, 36 insertions(+), 21 deletions(-) diff --git a/fs/bcachefs/btree_cache.c b/fs/bcachefs/btree_cache.c index 662f0f79b7af..7e74df9f194c 100644 --- a/fs/bcachefs/btree_cache.c +++ b/fs/bcachefs/btree_cache.c @@ -19,7 +19,7 @@ #define BTREE_CACHE_NOT_FREED_INCREMENT(counter) \ do { \ if (shrinker_counter) \ - bc->not_freed_##counter++; \ + bc->not_freed[BCH_BTREE_CACHE_NOT_FREED_##counter]++; \ } while (0) const char * const bch2_btree_node_flags[] = { @@ -270,8 +270,10 @@ static int __btree_node_reclaim(struct bch_fs *c, struct btree *b, bool flush, b if ((mask & BIT_ULL(b->c.btree_id)) && bbpos_cmp(bc->pinned_nodes_start, pos) < 0 && - bbpos_cmp(bc->pinned_nodes_end, pos) >= 0) + bbpos_cmp(bc->pinned_nodes_end, pos) >= 0) { + BTREE_CACHE_NOT_FREED_INCREMENT(pinned); return -BCH_ERR_ENOMEM_btree_node_reclaim; + } wait_on_io: if (b->flags & ((1U << BTREE_NODE_dirty)| @@ -433,7 +435,7 @@ restart: if (btree_node_accessed(b)) { clear_btree_node_accessed(b); - bc->not_freed_access_bit++; + bc->not_freed[BCH_BTREE_CACHE_NOT_FREED_access_bit]++; } else if (!btree_node_reclaim(c, b, true)) { freed++; btree_node_data_free(c, b); @@ -1344,6 +1346,13 @@ static void prt_btree_cache_line(struct printbuf *out, const struct bch_fs *c, prt_printf(out, " (%u)\n", nr); } +static const char * const bch2_btree_cache_not_freed_reasons_strs[] = { +#define x(n) #n, + BCH_BTREE_CACHE_NOT_FREED_REASONS() +#undef x + NULL +}; + void bch2_btree_cache_to_text(struct printbuf *out, const struct btree_cache *bc) { struct bch_fs *c = container_of(bc, struct bch_fs, btree_cache); @@ -1362,13 +1371,8 @@ void bch2_btree_cache_to_text(struct printbuf *out, const struct btree_cache *bc prt_newline(out); prt_printf(out, "freed:\t%u\n", bc->freed); prt_printf(out, "not freed:\n"); - prt_printf(out, " dirty\t%u\n", bc->not_freed_dirty); - prt_printf(out, " write in flight\t%u\n", bc->not_freed_write_in_flight); - prt_printf(out, " read in flight\t%u\n", bc->not_freed_read_in_flight); - prt_printf(out, " lock intent failed\t%u\n", bc->not_freed_lock_intent); - prt_printf(out, " lock write failed\t%u\n", bc->not_freed_lock_write); - prt_printf(out, " access bit\t%u\n", bc->not_freed_access_bit); - prt_printf(out, " no evict failed\t%u\n", bc->not_freed_noevict); - prt_printf(out, " write blocked\t%u\n", bc->not_freed_write_blocked); - prt_printf(out, " will make reachable\t%u\n", bc->not_freed_will_make_reachable); + + for (unsigned i = 0; i < ARRAY_SIZE(bc->not_freed); i++) + prt_printf(out, " %s\t%llu\n", + bch2_btree_cache_not_freed_reasons_strs[i], bc->not_freed[i]); } diff --git a/fs/bcachefs/btree_types.h b/fs/bcachefs/btree_types.h index 0df07929c545..c1ab824e1c34 100644 --- a/fs/bcachefs/btree_types.h +++ b/fs/bcachefs/btree_types.h @@ -138,6 +138,25 @@ struct btree { struct list_head list; }; +#define BCH_BTREE_CACHE_NOT_FREED_REASONS() \ + x(lock_intent) \ + x(lock_write) \ + x(dirty) \ + x(read_in_flight) \ + x(write_in_flight) \ + x(noevict) \ + x(write_blocked) \ + x(will_make_reachable) \ + x(access_bit) \ + x(pinned) \ + +enum bch_btree_cache_not_freed_reasons { +#define x(n) BCH_BTREE_CACHE_NOT_FREED_##n, + BCH_BTREE_CACHE_NOT_FREED_REASONS() +#undef x + BCH_BTREE_CACHE_NOT_FREED_REASONS_NR, +}; + struct btree_cache { struct rhashtable table; bool table_init_done; @@ -164,16 +183,8 @@ struct btree_cache { unsigned used; unsigned reserve; unsigned freed; - unsigned not_freed_lock_intent; - unsigned not_freed_lock_write; - unsigned not_freed_dirty; - unsigned not_freed_read_in_flight; - unsigned not_freed_write_in_flight; - unsigned not_freed_noevict; - unsigned not_freed_write_blocked; - unsigned not_freed_will_make_reachable; - unsigned not_freed_access_bit; atomic_t dirty; + u64 not_freed[BCH_BTREE_CACHE_NOT_FREED_REASONS_NR]; struct shrinker *shrink; unsigned used_by_btree[BTREE_ID_NR]; From b7d8092a1b3f0fe848eddacc225044a2a885bc02 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 1 Sep 2024 15:24:11 -0400 Subject: [PATCH 266/302] bcachefs: do_encrypt() now handles allocation failures convert to darray, and add a fallback when allocation fails Signed-off-by: Kent Overstreet --- fs/bcachefs/checksum.c | 48 ++++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/fs/bcachefs/checksum.c b/fs/bcachefs/checksum.c index e7208bf1974e..6e6cc1d04b1c 100644 --- a/fs/bcachefs/checksum.c +++ b/fs/bcachefs/checksum.c @@ -100,13 +100,12 @@ static inline int do_encrypt_sg(struct crypto_sync_skcipher *tfm, struct scatterlist *sg, size_t len) { SYNC_SKCIPHER_REQUEST_ON_STACK(req, tfm); - int ret; skcipher_request_set_sync_tfm(req, tfm); skcipher_request_set_callback(req, 0, NULL, NULL); skcipher_request_set_crypt(req, sg, sg, len, nonce.d); - ret = crypto_skcipher_encrypt(req); + int ret = crypto_skcipher_encrypt(req); if (ret) pr_err("got error %i from crypto_skcipher_encrypt()", ret); @@ -128,28 +127,41 @@ static inline int do_encrypt(struct crypto_sync_skcipher *tfm, len, offset_in_page(buf)); return do_encrypt_sg(tfm, nonce, &sg, len); } else { - unsigned pages = buf_pages(buf, len); - struct scatterlist *sg; - size_t orig_len = len; - int ret, i; + DARRAY_PREALLOCATED(struct scatterlist, 4) sgl; + size_t sgl_len = 0; + int ret; - sg = kmalloc_array(pages, sizeof(*sg), GFP_KERNEL); - if (!sg) - return -BCH_ERR_ENOMEM_do_encrypt; + darray_init(&sgl); - sg_init_table(sg, pages); - - for (i = 0; i < pages; i++) { + while (len) { unsigned offset = offset_in_page(buf); - unsigned pg_len = min_t(size_t, len, PAGE_SIZE - offset); + struct scatterlist sg = { + .page_link = (unsigned long) vmalloc_to_page(buf), + .offset = offset, + .length = min(len, PAGE_SIZE - offset), + }; - sg_set_page(sg + i, vmalloc_to_page(buf), pg_len, offset); - buf += pg_len; - len -= pg_len; + if (darray_push(&sgl, sg)) { + sg_mark_end(&darray_last(sgl)); + ret = do_encrypt_sg(tfm, nonce, sgl.data, sgl_len); + if (ret) + goto err; + + nonce = nonce_add(nonce, sgl_len); + sgl_len = 0; + sgl.nr = 0; + BUG_ON(darray_push(&sgl, sg)); + } + + buf += sg.length; + len -= sg.length; + sgl_len += sg.length; } - ret = do_encrypt_sg(tfm, nonce, sg, orig_len); - kfree(sg); + sg_mark_end(&darray_last(sgl)); + ret = do_encrypt_sg(tfm, nonce, sgl.data, sgl_len); +err: + darray_exit(&sgl); return ret; } } From af05633d4065673697c3f17de5db09938f96ebca Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 1 Sep 2024 15:33:17 -0400 Subject: [PATCH 267/302] bcachefs: convert __bch2_encrypt_bio() to darray like the previous patch, kill use of bare arrays; the encryption code likes to work in big batches, so this is a small performance improvement. Signed-off-by: Kent Overstreet --- fs/bcachefs/checksum.c | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/fs/bcachefs/checksum.c b/fs/bcachefs/checksum.c index 6e6cc1d04b1c..398177eb9c66 100644 --- a/fs/bcachefs/checksum.c +++ b/fs/bcachefs/checksum.c @@ -337,39 +337,42 @@ int __bch2_encrypt_bio(struct bch_fs *c, unsigned type, { struct bio_vec bv; struct bvec_iter iter; - struct scatterlist sgl[16], *sg = sgl; - size_t bytes = 0; + DARRAY_PREALLOCATED(struct scatterlist, 4) sgl; + size_t sgl_len = 0; int ret = 0; if (!bch2_csum_type_is_encryption(type)) return 0; - sg_init_table(sgl, ARRAY_SIZE(sgl)); + darray_init(&sgl); bio_for_each_segment(bv, bio, iter) { - if (sg == sgl + ARRAY_SIZE(sgl)) { - sg_mark_end(sg - 1); + struct scatterlist sg = { + .page_link = (unsigned long) bv.bv_page, + .offset = bv.bv_offset, + .length = bv.bv_len, + }; - ret = do_encrypt_sg(c->chacha20, nonce, sgl, bytes); + if (darray_push(&sgl, sg)) { + sg_mark_end(&darray_last(sgl)); + ret = do_encrypt_sg(c->chacha20, nonce, sgl.data, sgl_len); if (ret) - return ret; + goto err; - nonce = nonce_add(nonce, bytes); - bytes = 0; + nonce = nonce_add(nonce, sgl_len); + sgl_len = 0; + sgl.nr = 0; - sg_init_table(sgl, ARRAY_SIZE(sgl)); - sg = sgl; + BUG_ON(darray_push(&sgl, sg)); } - sg_set_page(sg++, bv.bv_page, bv.bv_len, bv.bv_offset); - bytes += bv.bv_len; - } - - if (sg != sgl) { - sg_mark_end(sg - 1); - return do_encrypt_sg(c->chacha20, nonce, sgl, bytes); + sgl_len += sg.length; } + sg_mark_end(&darray_last(sgl)); + ret = do_encrypt_sg(c->chacha20, nonce, sgl.data, sgl_len); +err: + darray_exit(&sgl); return ret; } From da2d20c98d32bbedc187f2b8fdb70d08fdeff057 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 1 Sep 2024 15:09:11 -0400 Subject: [PATCH 268/302] bcachefs: kill redundant is_vmalloc_addr() Signed-off-by: Kent Overstreet --- fs/bcachefs/checksum.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/fs/bcachefs/checksum.c b/fs/bcachefs/checksum.c index 398177eb9c66..ce8fc677bef9 100644 --- a/fs/bcachefs/checksum.c +++ b/fs/bcachefs/checksum.c @@ -117,14 +117,10 @@ static inline int do_encrypt(struct crypto_sync_skcipher *tfm, void *buf, size_t len) { if (!is_vmalloc_addr(buf)) { - struct scatterlist sg; + struct scatterlist sg = {}; - sg_init_table(&sg, 1); - sg_set_page(&sg, - is_vmalloc_addr(buf) - ? vmalloc_to_page(buf) - : virt_to_page(buf), - len, offset_in_page(buf)); + sg_mark_end(&sg); + sg_set_page(&sg, virt_to_page(buf), len, offset_in_page(buf)); return do_encrypt_sg(tfm, nonce, &sg, len); } else { DARRAY_PREALLOCATED(struct scatterlist, 4) sgl; From 2e95497e81010a0f409a7c0ab39404ef591371e9 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 1 Sep 2024 17:06:28 -0400 Subject: [PATCH 269/302] bcachefs: fix prototype to bch2_alloc_sectors_start_trans() Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_foreground.c | 28 +++++++++++++--------------- fs/bcachefs/alloc_foreground.h | 5 +++-- fs/bcachefs/btree_update_interior.c | 1 + fs/bcachefs/ec.c | 1 + 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/fs/bcachefs/alloc_foreground.c b/fs/bcachefs/alloc_foreground.c index 084b03b8c6e9..87400a4b2b8a 100644 --- a/fs/bcachefs/alloc_foreground.c +++ b/fs/bcachefs/alloc_foreground.c @@ -748,7 +748,6 @@ static int add_new_bucket(struct bch_fs *c, unsigned nr_replicas, unsigned *nr_effective, bool *have_cache, - unsigned flags, struct open_bucket *ob) { unsigned durability = ob_dev(c, ob)->mi.durability; @@ -775,7 +774,7 @@ int bch2_bucket_alloc_set_trans(struct btree_trans *trans, unsigned nr_replicas, unsigned *nr_effective, bool *have_cache, - unsigned flags, + enum bch_write_flags flags, enum bch_data_type data_type, enum bch_watermark watermark, struct closure *cl) @@ -815,7 +814,7 @@ int bch2_bucket_alloc_set_trans(struct btree_trans *trans, if (add_new_bucket(c, ptrs, devs_may_alloc, nr_replicas, nr_effective, - have_cache, flags, ob)) { + have_cache, ob)) { ret = 0; break; } @@ -841,7 +840,7 @@ static int bucket_alloc_from_stripe(struct btree_trans *trans, unsigned *nr_effective, bool *have_cache, enum bch_watermark watermark, - unsigned flags, + enum bch_write_flags flags, struct closure *cl) { struct bch_fs *c = trans->c; @@ -883,7 +882,7 @@ got_bucket: ret = add_new_bucket(c, ptrs, devs_may_alloc, nr_replicas, nr_effective, - have_cache, flags, ob); + have_cache, ob); out_put_head: bch2_ec_stripe_head_put(c, h); return ret; @@ -922,7 +921,7 @@ static int bucket_alloc_set_writepoint(struct bch_fs *c, unsigned nr_replicas, unsigned *nr_effective, bool *have_cache, - bool ec, unsigned flags) + bool ec) { struct open_buckets ptrs_skip = { .nr = 0 }; struct open_bucket *ob; @@ -934,7 +933,7 @@ static int bucket_alloc_set_writepoint(struct bch_fs *c, have_cache, ec, ob)) ret = add_new_bucket(c, ptrs, devs_may_alloc, nr_replicas, nr_effective, - have_cache, flags, ob); + have_cache, ob); else ob_push(c, &ptrs_skip, ob); } @@ -950,8 +949,7 @@ static int bucket_alloc_set_partial(struct bch_fs *c, unsigned nr_replicas, unsigned *nr_effective, bool *have_cache, bool ec, - enum bch_watermark watermark, - unsigned flags) + enum bch_watermark watermark) { int i, ret = 0; @@ -983,7 +981,7 @@ static int bucket_alloc_set_partial(struct bch_fs *c, ret = add_new_bucket(c, ptrs, devs_may_alloc, nr_replicas, nr_effective, - have_cache, flags, ob); + have_cache, ob); if (ret) break; } @@ -1003,7 +1001,7 @@ static int __open_bucket_add_buckets(struct btree_trans *trans, unsigned *nr_effective, bool *have_cache, enum bch_watermark watermark, - unsigned flags, + enum bch_write_flags flags, struct closure *_cl) { struct bch_fs *c = trans->c; @@ -1024,13 +1022,13 @@ static int __open_bucket_add_buckets(struct btree_trans *trans, ret = bucket_alloc_set_writepoint(c, ptrs, wp, &devs, nr_replicas, nr_effective, - have_cache, erasure_code, flags); + have_cache, erasure_code); if (ret) return ret; ret = bucket_alloc_set_partial(c, ptrs, wp, &devs, nr_replicas, nr_effective, - have_cache, erasure_code, watermark, flags); + have_cache, erasure_code, watermark); if (ret) return ret; @@ -1071,7 +1069,7 @@ static int open_bucket_add_buckets(struct btree_trans *trans, unsigned *nr_effective, bool *have_cache, enum bch_watermark watermark, - unsigned flags, + enum bch_write_flags flags, struct closure *cl) { int ret; @@ -1373,7 +1371,7 @@ int bch2_alloc_sectors_start_trans(struct btree_trans *trans, unsigned nr_replicas, unsigned nr_replicas_required, enum bch_watermark watermark, - unsigned flags, + enum bch_write_flags flags, struct closure *cl, struct write_point **wp_ret) { diff --git a/fs/bcachefs/alloc_foreground.h b/fs/bcachefs/alloc_foreground.h index 386d231ceca3..1a16fd5bd4f8 100644 --- a/fs/bcachefs/alloc_foreground.h +++ b/fs/bcachefs/alloc_foreground.h @@ -155,9 +155,10 @@ static inline bool bch2_bucket_is_open_safe(struct bch_fs *c, unsigned dev, u64 return ret; } +enum bch_write_flags; int bch2_bucket_alloc_set_trans(struct btree_trans *, struct open_buckets *, struct dev_stripe_state *, struct bch_devs_mask *, - unsigned, unsigned *, bool *, unsigned, + unsigned, unsigned *, bool *, enum bch_write_flags, enum bch_data_type, enum bch_watermark, struct closure *); @@ -167,7 +168,7 @@ int bch2_alloc_sectors_start_trans(struct btree_trans *, struct bch_devs_list *, unsigned, unsigned, enum bch_watermark, - unsigned, + enum bch_write_flags, struct closure *, struct write_point **); diff --git a/fs/bcachefs/btree_update_interior.c b/fs/bcachefs/btree_update_interior.c index 1433aefb47fd..35e49197bc57 100644 --- a/fs/bcachefs/btree_update_interior.c +++ b/fs/bcachefs/btree_update_interior.c @@ -16,6 +16,7 @@ #include "clock.h" #include "error.h" #include "extents.h" +#include "io_write.h" #include "journal.h" #include "journal_reclaim.h" #include "keylist.h" diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index 141a4c63142f..fa1be6a29d33 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -18,6 +18,7 @@ #include "ec.h" #include "error.h" #include "io_read.h" +#include "io_write.h" #include "keylist.h" #include "recovery.h" #include "replicas.h" From a977f3e162924ef82536ad23448364f2b3e9f026 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 1 Sep 2024 17:32:22 -0400 Subject: [PATCH 270/302] bcachefs: BCH_WRITE_ALLOC_NOWAIT no longer applies to open bucket allocation rebalance writes must be BCH_WRITE_ALLOC_NOWAIT because they don't allocate from the full filesystem - but we don't want spurious allocation failures due to open buckets. Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_foreground.c | 17 ++++++++++------- fs/bcachefs/errcode.h | 4 ++-- fs/bcachefs/io_write.c | 7 ++++--- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/fs/bcachefs/alloc_foreground.c b/fs/bcachefs/alloc_foreground.c index 87400a4b2b8a..7ab1200be95d 100644 --- a/fs/bcachefs/alloc_foreground.c +++ b/fs/bcachefs/alloc_foreground.c @@ -600,6 +600,7 @@ static struct open_bucket *bch2_bucket_alloc_trans(struct btree_trans *trans, enum bch_watermark watermark, enum bch_data_type data_type, struct closure *cl, + bool nowait, struct bch_dev_usage *usage) { struct bch_fs *c = trans->c; @@ -609,7 +610,7 @@ static struct open_bucket *bch2_bucket_alloc_trans(struct btree_trans *trans, struct bucket_alloc_state s = { .btree_bitmap = data_type == BCH_DATA_btree, }; - bool waiting = false; + bool waiting = nowait; again: bch2_dev_usage_read_fast(ca, usage); avail = dev_buckets_free(ca, *usage, watermark); @@ -685,7 +686,7 @@ struct open_bucket *bch2_bucket_alloc(struct bch_fs *c, struct bch_dev *ca, bch2_trans_do(c, NULL, NULL, 0, PTR_ERR_OR_ZERO(ob = bch2_bucket_alloc_trans(trans, ca, watermark, - data_type, cl, &usage))); + data_type, cl, false, &usage))); return ob; } @@ -800,7 +801,8 @@ int bch2_bucket_alloc_set_trans(struct btree_trans *trans, continue; } - ob = bch2_bucket_alloc_trans(trans, ca, watermark, data_type, cl, &usage); + ob = bch2_bucket_alloc_trans(trans, ca, watermark, data_type, + cl, flags & BCH_WRITE_ALLOC_NOWAIT, &usage); if (!IS_ERR(ob)) bch2_dev_stripe_increment_inlined(ca, stripe, &usage); bch2_dev_put(ca); @@ -1493,11 +1495,12 @@ err: try_decrease_writepoints(trans, write_points_nr)) goto retry; - if (bch2_err_matches(ret, BCH_ERR_open_buckets_empty) || + if (cl && bch2_err_matches(ret, BCH_ERR_open_buckets_empty)) + ret = -BCH_ERR_bucket_alloc_blocked; + + if (cl && !(flags & BCH_WRITE_ALLOC_NOWAIT) && bch2_err_matches(ret, BCH_ERR_freelist_empty)) - return cl - ? -BCH_ERR_bucket_alloc_blocked - : -BCH_ERR_ENOSPC_bucket_alloc; + ret = -BCH_ERR_bucket_alloc_blocked; return ret; } diff --git a/fs/bcachefs/errcode.h b/fs/bcachefs/errcode.h index 742dcdd3e5d7..96be8d600ca0 100644 --- a/fs/bcachefs/errcode.h +++ b/fs/bcachefs/errcode.h @@ -119,8 +119,8 @@ x(EEXIST, EEXIST_str_hash_set) \ x(EEXIST, EEXIST_discard_in_flight_add) \ x(EEXIST, EEXIST_subvolume_create) \ - x(0, open_buckets_empty) \ - x(0, freelist_empty) \ + x(ENOSPC, open_buckets_empty) \ + x(ENOSPC, freelist_empty) \ x(BCH_ERR_freelist_empty, no_buckets_found) \ x(0, transaction_restart) \ x(BCH_ERR_transaction_restart, transaction_restart_fault_inject) \ diff --git a/fs/bcachefs/io_write.c b/fs/bcachefs/io_write.c index 1d4761d15002..d3b5be7fd9bf 100644 --- a/fs/bcachefs/io_write.c +++ b/fs/bcachefs/io_write.c @@ -1447,9 +1447,7 @@ again: op->nr_replicas_required, op->watermark, op->flags, - (op->flags & (BCH_WRITE_ALLOC_NOWAIT| - BCH_WRITE_ONLY_SPECIFIED_DEVS)) - ? NULL : &op->cl, &wp)); + &op->cl, &wp)); if (unlikely(ret)) { if (bch2_err_matches(ret, BCH_ERR_operation_blocked)) break; @@ -1592,6 +1590,9 @@ CLOSURE_CALLBACK(bch2_write) BUG_ON(!op->write_point.v); BUG_ON(bkey_eq(op->pos, POS_MAX)); + if (op->flags & BCH_WRITE_ONLY_SPECIFIED_DEVS) + op->flags |= BCH_WRITE_ALLOC_NOWAIT; + op->nr_replicas_required = min_t(unsigned, op->nr_replicas_required, op->nr_replicas); op->start_time = local_clock(); bch2_keylist_init(&op->insert_keys, op->inline_keys); From d90c8acd351c042f4bf1676e71c3f80fb3492403 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 1 Sep 2024 16:55:35 -0400 Subject: [PATCH 271/302] bcachefs: rebalance writes use BCH_WRITE_ONLY_SPECIFIED_DEVS this was an oversight: rebalance is moving data to a specific device, so we don't want it falling back to the full filesystem Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_foreground.c | 2 -- fs/bcachefs/rebalance.c | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/fs/bcachefs/alloc_foreground.c b/fs/bcachefs/alloc_foreground.c index 7ab1200be95d..bf8fb9cd650b 100644 --- a/fs/bcachefs/alloc_foreground.c +++ b/fs/bcachefs/alloc_foreground.c @@ -1389,8 +1389,6 @@ int bch2_alloc_sectors_start_trans(struct btree_trans *trans, if (!IS_ENABLED(CONFIG_BCACHEFS_ERASURE_CODING)) erasure_code = false; - BUG_ON(flags & BCH_WRITE_ONLY_SPECIFIED_DEVS); - BUG_ON(!nr_replicas || !nr_replicas_required); retry: ptrs.nr = 0; diff --git a/fs/bcachefs/rebalance.c b/fs/bcachefs/rebalance.c index cf81e5128c3a..2d299a37cf07 100644 --- a/fs/bcachefs/rebalance.c +++ b/fs/bcachefs/rebalance.c @@ -13,6 +13,7 @@ #include "errcode.h" #include "error.h" #include "inode.h" +#include "io_write.h" #include "move.h" #include "rebalance.h" #include "subvolume.h" @@ -156,6 +157,7 @@ static struct bkey_s_c next_rebalance_extent(struct btree_trans *trans, data_opts->rewrite_ptrs = bch2_bkey_ptrs_need_rebalance(c, k, r->target, r->compression); data_opts->target = r->target; + data_opts->write_flags |= BCH_WRITE_ONLY_SPECIFIED_DEVS; if (!data_opts->rewrite_ptrs) { /* @@ -263,6 +265,7 @@ static bool rebalance_pred(struct bch_fs *c, void *arg, data_opts->rewrite_ptrs = bch2_bkey_ptrs_need_rebalance(c, k, target, compression); data_opts->target = target; + data_opts->write_flags |= BCH_WRITE_ONLY_SPECIFIED_DEVS; return data_opts->rewrite_ptrs != 0; } From 4645855df01eda7c421c182515215682e0451b9b Mon Sep 17 00:00:00 2001 From: Sasha Finkelstein Date: Sun, 18 Aug 2024 19:09:02 +0200 Subject: [PATCH 272/302] bcachefs: Hook up RENAME_WHITEOUT in rename. This is needed for overlayfs, which is used by container managers. Signed-off-by: Sasha Finkelstein Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_iter.h | 6 +++++ fs/bcachefs/fs-common.c | 5 ++-- fs/bcachefs/fs.c | 53 +++++++++++++++++++++++++++++++--------- fs/bcachefs/str_hash.h | 2 +- 4 files changed, 52 insertions(+), 14 deletions(-) diff --git a/fs/bcachefs/btree_iter.h b/fs/bcachefs/btree_iter.h index 6d87e57745da..78e63ad7d380 100644 --- a/fs/bcachefs/btree_iter.h +++ b/fs/bcachefs/btree_iter.h @@ -529,6 +529,12 @@ void bch2_set_btree_iter_dontneed(struct btree_iter *); void *__bch2_trans_kmalloc(struct btree_trans *, size_t); +/** + * bch2_trans_kmalloc - allocate memory for use by the current transaction + * + * Must be called after bch2_trans_begin, which on second and further calls + * frees all memory allocated in this transaction + */ static inline void *bch2_trans_kmalloc(struct btree_trans *trans, size_t size) { size = roundup(size, 8); diff --git a/fs/bcachefs/fs-common.c b/fs/bcachefs/fs-common.c index 508d029ac53d..7e10a9ddcfd9 100644 --- a/fs/bcachefs/fs-common.c +++ b/fs/bcachefs/fs-common.c @@ -42,7 +42,8 @@ int bch2_create_trans(struct btree_trans *trans, if (ret) goto err; - ret = bch2_inode_peek(trans, &dir_iter, dir_u, dir, BTREE_ITER_intent); + ret = bch2_inode_peek(trans, &dir_iter, dir_u, dir, + BTREE_ITER_intent|BTREE_ITER_with_updates); if (ret) goto err; @@ -163,7 +164,7 @@ int bch2_create_trans(struct btree_trans *trans, name, dir_target, &dir_offset, - STR_HASH_must_create); + STR_HASH_must_create|BTREE_ITER_with_updates); if (ret) goto err; diff --git a/fs/bcachefs/fs.c b/fs/bcachefs/fs.c index 866ce1f1a6d1..dfadfb82399f 100644 --- a/fs/bcachefs/fs.c +++ b/fs/bcachefs/fs.c @@ -736,15 +736,16 @@ static int bch2_rename2(struct mnt_idmap *idmap, struct bch_inode_info *src_inode = to_bch_ei(src_dentry->d_inode); struct bch_inode_info *dst_inode = to_bch_ei(dst_dentry->d_inode); struct bch_inode_unpacked dst_dir_u, src_dir_u; - struct bch_inode_unpacked src_inode_u, dst_inode_u; + struct bch_inode_unpacked src_inode_u, dst_inode_u, *whiteout_inode_u; struct btree_trans *trans; enum bch_rename_mode mode = flags & RENAME_EXCHANGE ? BCH_RENAME_EXCHANGE : dst_dentry->d_inode ? BCH_RENAME_OVERWRITE : BCH_RENAME; + bool whiteout = !!(flags & RENAME_WHITEOUT); int ret; - if (flags & ~(RENAME_NOREPLACE|RENAME_EXCHANGE)) + if (flags & ~(RENAME_NOREPLACE|RENAME_EXCHANGE|RENAME_WHITEOUT)) return -EINVAL; if (mode == BCH_RENAME_OVERWRITE) { @@ -785,18 +786,48 @@ static int bch2_rename2(struct mnt_idmap *idmap, if (ret) goto err; } +retry: + bch2_trans_begin(trans); - ret = commit_do(trans, NULL, NULL, 0, - bch2_rename_trans(trans, - inode_inum(src_dir), &src_dir_u, - inode_inum(dst_dir), &dst_dir_u, - &src_inode_u, - &dst_inode_u, - &src_dentry->d_name, - &dst_dentry->d_name, - mode)); + ret = bch2_rename_trans(trans, + inode_inum(src_dir), &src_dir_u, + inode_inum(dst_dir), &dst_dir_u, + &src_inode_u, + &dst_inode_u, + &src_dentry->d_name, + &dst_dentry->d_name, + mode); if (unlikely(ret)) + goto err_tx_restart; + + if (whiteout) { + whiteout_inode_u = bch2_trans_kmalloc_nomemzero(trans, sizeof(*whiteout_inode_u)); + ret = PTR_ERR_OR_ZERO(whiteout_inode_u); + if (unlikely(ret)) + goto err_tx_restart; + bch2_inode_init_early(c, whiteout_inode_u); + + ret = bch2_create_trans(trans, + inode_inum(src_dir), &src_dir_u, + whiteout_inode_u, + &src_dentry->d_name, + from_kuid(i_user_ns(&src_dir->v), current_fsuid()), + from_kgid(i_user_ns(&src_dir->v), current_fsgid()), + S_IFCHR|WHITEOUT_MODE, 0, + NULL, NULL, (subvol_inum) { 0 }, 0) ?: + bch2_quota_acct(c, bch_qid(whiteout_inode_u), Q_INO, 1, + KEY_TYPE_QUOTA_PREALLOC); + if (unlikely(ret)) + goto err_tx_restart; + } + + ret = bch2_trans_commit(trans, NULL, NULL, 0); + if (unlikely(ret)) { +err_tx_restart: + if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) + goto retry; goto err; + } BUG_ON(src_inode->v.i_ino != src_inode_u.bi_inum); BUG_ON(dst_inode && diff --git a/fs/bcachefs/str_hash.h b/fs/bcachefs/str_hash.h index c8c266cb5797..215eed4cce6d 100644 --- a/fs/bcachefs/str_hash.h +++ b/fs/bcachefs/str_hash.h @@ -270,7 +270,7 @@ int bch2_hash_set_in_snapshot(struct btree_trans *trans, desc.hash_bkey(info, bkey_i_to_s_c(insert)), snapshot), POS(insert->k.p.inode, U64_MAX), - BTREE_ITER_slots|BTREE_ITER_intent, k, ret) { + BTREE_ITER_slots|BTREE_ITER_intent|flags, k, ret) { if (is_visible_key(desc, inum, k)) { if (!desc.cmp_bkey(k, bkey_i_to_s_c(insert))) goto found; From 895fbf1cf02f0c913bb443e9ae18a230bbd174c1 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 3 Sep 2024 17:42:53 -0400 Subject: [PATCH 273/302] bcachefs: Use __GFP_ACCOUNT for reclaimable memory Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_cache.c | 2 ++ fs/bcachefs/btree_key_cache.c | 2 ++ 2 files changed, 4 insertions(+) diff --git a/fs/bcachefs/btree_cache.c b/fs/bcachefs/btree_cache.c index 7e74df9f194c..02e6e609e377 100644 --- a/fs/bcachefs/btree_cache.c +++ b/fs/bcachefs/btree_cache.c @@ -102,6 +102,8 @@ static int btree_node_data_alloc(struct bch_fs *c, struct btree *b, gfp_t gfp) { BUG_ON(b->data || b->aux_data); + gfp |= __GFP_ACCOUNT|__GFP_RECLAIMABLE; + b->data = kvmalloc(btree_buf_bytes(b), gfp); if (!b->data) return -BCH_ERR_ENOMEM_btree_node_mem_alloc; diff --git a/fs/bcachefs/btree_key_cache.c b/fs/bcachefs/btree_key_cache.c index 2e49ca71194f..244610b1d0b5 100644 --- a/fs/bcachefs/btree_key_cache.c +++ b/fs/bcachefs/btree_key_cache.c @@ -118,6 +118,8 @@ static void bkey_cached_free(struct btree_key_cache *bc, static struct bkey_cached *__bkey_cached_alloc(unsigned key_u64s, gfp_t gfp) { + gfp |= __GFP_ACCOUNT|__GFP_RECLAIMABLE; + struct bkey_cached *ck = kmem_cache_zalloc(bch2_key_cache, gfp); if (unlikely(!ck)) return NULL; From ff7f756f2be5b37e323ff8215e500ee564551db9 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 4 Sep 2024 15:30:48 -0400 Subject: [PATCH 274/302] bcachefs: Use mm_account_reclaimed_pages() when freeing btree nodes When freeing in a shrinker callback, we need to notify memory reclaim, so it knows forward progress has been made. Normally this is done in e.g. slab code, but we're not freeing through slab - or rather we are, but these allocations are big, and use the kmalloc_large() path. This is really a bug in the slub code, but we're working around it here for now. Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_cache.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/fs/bcachefs/btree_cache.c b/fs/bcachefs/btree_cache.c index 02e6e609e377..9a2e6c6ea82c 100644 --- a/fs/bcachefs/btree_cache.c +++ b/fs/bcachefs/btree_cache.c @@ -15,6 +15,7 @@ #include #include +#include #define BTREE_CACHE_NOT_FREED_INCREMENT(counter) \ do { \ @@ -63,6 +64,16 @@ static void btree_node_data_free(struct bch_fs *c, struct btree *b) { struct btree_cache *bc = &c->btree_cache; + /* + * This should really be done in slub/vmalloc, but we're using the + * kmalloc_large() path, so we're working around a slub bug by doing + * this here: + */ + if (b->data) + mm_account_reclaimed_pages(btree_buf_bytes(b) / PAGE_SIZE); + if (b->aux_data) + mm_account_reclaimed_pages(btree_aux_data_bytes(b) / PAGE_SIZE); + EBUG_ON(btree_node_write_in_flight(b)); clear_btree_node_just_written(b); From 4f19a60c3226733917aac815fc156a0ea03b335e Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 1 Sep 2024 22:39:42 -0400 Subject: [PATCH 275/302] bcachefs: Options for recovery_passes, recovery_passes_exclude This adds mount options for specifying recovery passes to run, or exclude; the immediate need for this is that backpointers fsck is having trouble completing, so we need a way to skip it. Signed-off-by: Kent Overstreet --- fs/bcachefs/bcachefs.h | 2 -- fs/bcachefs/btree_io.c | 2 +- fs/bcachefs/btree_update_interior.c | 2 +- fs/bcachefs/opts.c | 3 +++ fs/bcachefs/opts.h | 10 ++++++++++ fs/bcachefs/recovery.c | 22 +++++++++++----------- fs/bcachefs/recovery_passes.c | 10 ++++++---- fs/bcachefs/util.c | 2 +- 8 files changed, 33 insertions(+), 20 deletions(-) diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h index ad4df0ad26eb..b2475b9555ac 100644 --- a/fs/bcachefs/bcachefs.h +++ b/fs/bcachefs/bcachefs.h @@ -1045,8 +1045,6 @@ struct bch_fs { * for signaling to the toplevel code which pass we want to run now. */ enum bch_recovery_pass curr_recovery_pass; - /* bitmap of explicitly enabled recovery passes: */ - u64 recovery_passes_explicit; /* bitmask of recovery passes that we actually ran */ u64 recovery_passes_complete; /* never rewinds version of curr_recovery_pass */ diff --git a/fs/bcachefs/btree_io.c b/fs/bcachefs/btree_io.c index 56ea9a77cd4a..18cff98650de 100644 --- a/fs/bcachefs/btree_io.c +++ b/fs/bcachefs/btree_io.c @@ -1666,7 +1666,7 @@ void bch2_btree_node_read(struct btree_trans *trans, struct btree *b, bch2_btree_pos_to_text(&buf, c, b); bch_err_ratelimited(c, "%s", buf.buf); - if (c->recovery_passes_explicit & BIT_ULL(BCH_RECOVERY_PASS_check_topology) && + if (c->opts.recovery_passes & BIT_ULL(BCH_RECOVERY_PASS_check_topology) && c->curr_recovery_pass > BCH_RECOVERY_PASS_check_topology) bch2_fatal_error(c); diff --git a/fs/bcachefs/btree_update_interior.c b/fs/bcachefs/btree_update_interior.c index 35e49197bc57..24e445574ab8 100644 --- a/fs/bcachefs/btree_update_interior.c +++ b/fs/bcachefs/btree_update_interior.c @@ -146,7 +146,7 @@ fsck_err: printbuf_exit(&buf); return ret; topology_repair: - if ((c->recovery_passes_explicit & BIT_ULL(BCH_RECOVERY_PASS_check_topology)) && + if ((c->opts.recovery_passes & BIT_ULL(BCH_RECOVERY_PASS_check_topology)) && c->curr_recovery_pass > BCH_RECOVERY_PASS_check_topology) { bch2_inconsistent_error(c); ret = -BCH_ERR_btree_need_topology_repair; diff --git a/fs/bcachefs/opts.c b/fs/bcachefs/opts.c index 0770aebef6d8..9cb9b05db6d1 100644 --- a/fs/bcachefs/opts.c +++ b/fs/bcachefs/opts.c @@ -432,6 +432,9 @@ void bch2_opt_to_text(struct printbuf *out, else prt_str(out, opt->choices[v]); break; + case BCH_OPT_BITFIELD: + prt_bitflags(out, opt->choices, v); + break; case BCH_OPT_FN: opt->fn.to_text(out, c, sb, v); break; diff --git a/fs/bcachefs/opts.h b/fs/bcachefs/opts.h index 3d83bcdca69d..f2ae13f0fbad 100644 --- a/fs/bcachefs/opts.h +++ b/fs/bcachefs/opts.h @@ -373,6 +373,16 @@ enum fsck_err_opts { OPT_BOOL(), \ BCH2_NO_SB_OPT, false, \ NULL, "Exit recovery immediately prior to journal replay")\ + x(recovery_passes, u64, \ + OPT_FS|OPT_MOUNT, \ + OPT_BITFIELD(bch2_recovery_passes), \ + BCH2_NO_SB_OPT, 0, \ + NULL, "Recovery passes to run explicitly") \ + x(recovery_passes_exclude, u64, \ + OPT_FS|OPT_MOUNT, \ + OPT_BITFIELD(bch2_recovery_passes), \ + BCH2_NO_SB_OPT, 0, \ + NULL, "Recovery passes to exclude") \ x(recovery_pass_last, u8, \ OPT_FS|OPT_MOUNT, \ OPT_STR_NOLIMIT(bch2_recovery_passes), \ diff --git a/fs/bcachefs/recovery.c b/fs/bcachefs/recovery.c index 36de1c6fe8c3..be1e7ca4362f 100644 --- a/fs/bcachefs/recovery.c +++ b/fs/bcachefs/recovery.c @@ -97,7 +97,7 @@ static void bch2_reconstruct_alloc(struct bch_fs *c) bch2_write_super(c); mutex_unlock(&c->sb_lock); - c->recovery_passes_explicit |= bch2_recovery_passes_from_stable(le64_to_cpu(ext->recovery_passes_required[0])); + c->opts.recovery_passes |= bch2_recovery_passes_from_stable(le64_to_cpu(ext->recovery_passes_required[0])); bch2_shoot_down_journal_keys(c, BTREE_ID_alloc, @@ -525,17 +525,17 @@ static int read_btree_roots(struct bch_fs *c) "error reading btree root %s l=%u: %s", bch2_btree_id_str(i), r->level, bch2_err_str(ret))) { if (btree_id_is_alloc(i)) { - c->recovery_passes_explicit |= BIT_ULL(BCH_RECOVERY_PASS_check_allocations); - c->recovery_passes_explicit |= BIT_ULL(BCH_RECOVERY_PASS_check_alloc_info); - c->recovery_passes_explicit |= BIT_ULL(BCH_RECOVERY_PASS_check_lrus); - c->recovery_passes_explicit |= BIT_ULL(BCH_RECOVERY_PASS_check_extents_to_backpointers); - c->recovery_passes_explicit |= BIT_ULL(BCH_RECOVERY_PASS_check_alloc_to_lru_refs); + c->opts.recovery_passes |= BIT_ULL(BCH_RECOVERY_PASS_check_allocations); + c->opts.recovery_passes |= BIT_ULL(BCH_RECOVERY_PASS_check_alloc_info); + c->opts.recovery_passes |= BIT_ULL(BCH_RECOVERY_PASS_check_lrus); + c->opts.recovery_passes |= BIT_ULL(BCH_RECOVERY_PASS_check_extents_to_backpointers); + c->opts.recovery_passes |= BIT_ULL(BCH_RECOVERY_PASS_check_alloc_to_lru_refs); c->sb.compat &= ~(1ULL << BCH_COMPAT_alloc_info); r->error = 0; - } else if (!(c->recovery_passes_explicit & BIT_ULL(BCH_RECOVERY_PASS_scan_for_btree_nodes))) { + } else if (!(c->opts.recovery_passes & BIT_ULL(BCH_RECOVERY_PASS_scan_for_btree_nodes))) { bch_info(c, "will run btree node scan"); - c->recovery_passes_explicit |= BIT_ULL(BCH_RECOVERY_PASS_scan_for_btree_nodes); - c->recovery_passes_explicit |= BIT_ULL(BCH_RECOVERY_PASS_check_topology); + c->opts.recovery_passes |= BIT_ULL(BCH_RECOVERY_PASS_scan_for_btree_nodes); + c->opts.recovery_passes |= BIT_ULL(BCH_RECOVERY_PASS_check_topology); } ret = 0; @@ -706,14 +706,14 @@ int bch2_fs_recovery(struct bch_fs *c) if (check_version_upgrade(c)) write_sb = true; - c->recovery_passes_explicit |= bch2_recovery_passes_from_stable(le64_to_cpu(ext->recovery_passes_required[0])); + c->opts.recovery_passes |= bch2_recovery_passes_from_stable(le64_to_cpu(ext->recovery_passes_required[0])); if (write_sb) bch2_write_super(c); mutex_unlock(&c->sb_lock); if (c->opts.fsck && IS_ENABLED(CONFIG_BCACHEFS_DEBUG)) - c->recovery_passes_explicit |= BIT_ULL(BCH_RECOVERY_PASS_check_topology); + c->opts.recovery_passes |= BIT_ULL(BCH_RECOVERY_PASS_check_topology); if (c->opts.fsck) set_bit(BCH_FS_fsck_running, &c->flags); diff --git a/fs/bcachefs/recovery_passes.c b/fs/bcachefs/recovery_passes.c index 73339a0a3111..735b8adc8f9d 100644 --- a/fs/bcachefs/recovery_passes.c +++ b/fs/bcachefs/recovery_passes.c @@ -40,7 +40,7 @@ static int bch2_set_may_go_rw(struct bch_fs *c) set_bit(BCH_FS_may_go_rw, &c->flags); - if (keys->nr || c->opts.fsck || !c->sb.clean || c->recovery_passes_explicit) + if (keys->nr || c->opts.fsck || !c->sb.clean || c->opts.recovery_passes) return bch2_fs_read_write_early(c); return 0; } @@ -97,14 +97,14 @@ u64 bch2_recovery_passes_from_stable(u64 v) int bch2_run_explicit_recovery_pass(struct bch_fs *c, enum bch_recovery_pass pass) { - if (c->recovery_passes_explicit & BIT_ULL(pass)) + if (c->opts.recovery_passes & BIT_ULL(pass)) return 0; bch_info(c, "running explicit recovery pass %s (%u), currently at %s (%u)", bch2_recovery_passes[pass], pass, bch2_recovery_passes[c->curr_recovery_pass], c->curr_recovery_pass); - c->recovery_passes_explicit |= BIT_ULL(pass); + c->opts.recovery_passes |= BIT_ULL(pass); if (c->curr_recovery_pass >= pass) { c->curr_recovery_pass = pass; @@ -161,7 +161,9 @@ static bool should_run_recovery_pass(struct bch_fs *c, enum bch_recovery_pass pa { struct recovery_pass_fn *p = recovery_pass_fns + pass; - if (c->recovery_passes_explicit & BIT_ULL(pass)) + if (c->opts.recovery_passes_exclude & BIT_ULL(pass)) + return false; + if (c->opts.recovery_passes & BIT_ULL(pass)) return true; if ((p->when & PASS_FSCK) && c->opts.fsck) return true; diff --git a/fs/bcachefs/util.c b/fs/bcachefs/util.c index 8b4e8395a908..42f565c76181 100644 --- a/fs/bcachefs/util.c +++ b/fs/bcachefs/util.c @@ -214,7 +214,7 @@ u64 bch2_read_flag_list(const char *opt, const char * const list[]) s = strim(d); - while ((p = strsep(&s, ","))) { + while ((p = strsep(&s, ",;"))) { int flag = match_string(list, -1, p); if (flag < 0) { From 8ed4ba3663318f1e89cec1a5e6a13b84e91fc012 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 4 Sep 2024 17:51:16 -0400 Subject: [PATCH 276/302] bcachefs: Move tabstop setup to bch2_dev_usage_to_text() No reason for it not to be where it's needed. Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_foreground.c | 7 ------- fs/bcachefs/buckets.c | 9 +++++++++ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/fs/bcachefs/alloc_foreground.c b/fs/bcachefs/alloc_foreground.c index bf8fb9cd650b..d0e0b56892e3 100644 --- a/fs/bcachefs/alloc_foreground.c +++ b/fs/bcachefs/alloc_foreground.c @@ -1729,13 +1729,6 @@ void bch2_dev_alloc_debug_to_text(struct printbuf *out, struct bch_dev *ca) for (unsigned i = 0; i < ARRAY_SIZE(c->open_buckets); i++) nr[c->open_buckets[i].data_type]++; - printbuf_tabstops_reset(out); - printbuf_tabstop_push(out, 12); - printbuf_tabstop_push(out, 16); - printbuf_tabstop_push(out, 16); - printbuf_tabstop_push(out, 16); - printbuf_tabstop_push(out, 16); - bch2_dev_usage_to_text(out, ca, &stats); prt_newline(out); diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c index 4a0f55dc78c4..4abd22d7e3f8 100644 --- a/fs/bcachefs/buckets.c +++ b/fs/bcachefs/buckets.c @@ -75,6 +75,15 @@ void bch2_dev_usage_to_text(struct printbuf *out, struct bch_dev *ca, struct bch_dev_usage *usage) { + if (out->nr_tabstops < 5) { + printbuf_tabstops_reset(out); + printbuf_tabstop_push(out, 12); + printbuf_tabstop_push(out, 16); + printbuf_tabstop_push(out, 16); + printbuf_tabstop_push(out, 16); + printbuf_tabstop_push(out, 16); + } + prt_printf(out, "\tbuckets\rsectors\rfragmented\r\n"); for (unsigned i = 0; i < BCH_DATA_NR; i++) { From 6b812f1dcebfee45d9ef6856404fa666797ac979 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 1 Sep 2024 17:56:27 -0400 Subject: [PATCH 277/302] bcachefs: bch2_dev_remove_alloc() -> alloc_background.c Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_background.c | 29 +++++++++++++++++++++++++++++ fs/bcachefs/alloc_background.h | 1 + fs/bcachefs/super.c | 27 --------------------------- 3 files changed, 30 insertions(+), 27 deletions(-) diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c index a6b9fe7038f3..d4f0b62e3138 100644 --- a/fs/bcachefs/alloc_background.c +++ b/fs/bcachefs/alloc_background.c @@ -2298,6 +2298,35 @@ int bch2_fs_freespace_init(struct bch_fs *c) return 0; } +/* device removal */ + +int bch2_dev_remove_alloc(struct bch_fs *c, struct bch_dev *ca) +{ + struct bpos start = POS(ca->dev_idx, 0); + struct bpos end = POS(ca->dev_idx, U64_MAX); + int ret; + + /* + * We clear the LRU and need_discard btrees first so that we don't race + * with bch2_do_invalidates() and bch2_do_discards() + */ + ret = bch2_btree_delete_range(c, BTREE_ID_lru, start, end, + BTREE_TRIGGER_norun, NULL) ?: + bch2_btree_delete_range(c, BTREE_ID_need_discard, start, end, + BTREE_TRIGGER_norun, NULL) ?: + bch2_btree_delete_range(c, BTREE_ID_freespace, start, end, + BTREE_TRIGGER_norun, NULL) ?: + bch2_btree_delete_range(c, BTREE_ID_backpointers, start, end, + BTREE_TRIGGER_norun, NULL) ?: + bch2_btree_delete_range(c, BTREE_ID_alloc, start, end, + BTREE_TRIGGER_norun, NULL) ?: + bch2_btree_delete_range(c, BTREE_ID_bucket_gens, start, end, + BTREE_TRIGGER_norun, NULL) ?: + bch2_dev_usage_remove(c, ca->dev_idx); + bch_err_msg(ca, ret, "removing dev alloc info"); + return ret; +} + /* Bucket IO clocks: */ int bch2_bucket_io_time_reset(struct btree_trans *trans, unsigned dev, diff --git a/fs/bcachefs/alloc_background.h b/fs/bcachefs/alloc_background.h index fd790b03fbe1..577f823a0207 100644 --- a/fs/bcachefs/alloc_background.h +++ b/fs/bcachefs/alloc_background.h @@ -338,6 +338,7 @@ static inline const struct bch_backpointer *alloc_v4_backpointers_c(const struct int bch2_dev_freespace_init(struct bch_fs *, struct bch_dev *, u64, u64); int bch2_fs_freespace_init(struct bch_fs *); +int bch2_dev_remove_alloc(struct bch_fs *, struct bch_dev *); void bch2_recalc_capacity(struct bch_fs *); u64 bch2_min_rw_member_capacity(struct bch_fs *); diff --git a/fs/bcachefs/super.c b/fs/bcachefs/super.c index d8adf465f8b8..7dba46e97fbd 100644 --- a/fs/bcachefs/super.c +++ b/fs/bcachefs/super.c @@ -1592,33 +1592,6 @@ int bch2_dev_set_state(struct bch_fs *c, struct bch_dev *ca, /* Device add/removal: */ -static int bch2_dev_remove_alloc(struct bch_fs *c, struct bch_dev *ca) -{ - struct bpos start = POS(ca->dev_idx, 0); - struct bpos end = POS(ca->dev_idx, U64_MAX); - int ret; - - /* - * We clear the LRU and need_discard btrees first so that we don't race - * with bch2_do_invalidates() and bch2_do_discards() - */ - ret = bch2_btree_delete_range(c, BTREE_ID_lru, start, end, - BTREE_TRIGGER_norun, NULL) ?: - bch2_btree_delete_range(c, BTREE_ID_need_discard, start, end, - BTREE_TRIGGER_norun, NULL) ?: - bch2_btree_delete_range(c, BTREE_ID_freespace, start, end, - BTREE_TRIGGER_norun, NULL) ?: - bch2_btree_delete_range(c, BTREE_ID_backpointers, start, end, - BTREE_TRIGGER_norun, NULL) ?: - bch2_btree_delete_range(c, BTREE_ID_alloc, start, end, - BTREE_TRIGGER_norun, NULL) ?: - bch2_btree_delete_range(c, BTREE_ID_bucket_gens, start, end, - BTREE_TRIGGER_norun, NULL) ?: - bch2_dev_usage_remove(c, ca->dev_idx); - bch_err_msg(c, ret, "removing dev alloc info"); - return ret; -} - int bch2_dev_remove(struct bch_fs *c, struct bch_dev *ca, int flags) { struct bch_member *m; From 17405279e82aaa302e134655f74e713a0968a16c Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 1 Sep 2024 18:08:25 -0400 Subject: [PATCH 278/302] bcachefs: bch2_sb_member_alloc() refactoring Signed-off-by: Kent Overstreet --- fs/bcachefs/sb-members.c | 44 +++++++++++++++++++++++++++++++++ fs/bcachefs/sb-members.h | 2 ++ fs/bcachefs/super.c | 53 ++++++---------------------------------- 3 files changed, 53 insertions(+), 46 deletions(-) diff --git a/fs/bcachefs/sb-members.c b/fs/bcachefs/sb-members.c index 7aa5067e09c5..2aad62961938 100644 --- a/fs/bcachefs/sb-members.c +++ b/fs/bcachefs/sb-members.c @@ -474,3 +474,47 @@ unsigned bch2_sb_nr_devices(const struct bch_sb *sb) nr += bch2_member_exists((struct bch_sb *) sb, i); return nr; } + +int bch2_sb_member_alloc(struct bch_fs *c) +{ + unsigned dev_idx = c->sb.nr_devices; + if (dev_idx < BCH_SB_MEMBERS_MAX) + goto have_slot; + + int best = -1; + u64 best_last_mount = 0; + for (dev_idx = 0; dev_idx < BCH_SB_MEMBERS_MAX; dev_idx++) { + /* eventually BCH_SB_MEMBERS_MAX will be raised */ + if (dev_idx == BCH_SB_MEMBER_INVALID) + continue; + + struct bch_member m = bch2_sb_member_get(c->disk_sb.sb, dev_idx); + if (bch2_member_alive(&m)) + continue; + + u64 last_mount = le64_to_cpu(m.last_mount); + if (best < 0 || last_mount < best_last_mount) { + best = dev_idx; + best_last_mount = last_mount; + } + } + if (best >= 0) { + dev_idx = best; + goto have_slot; + } + + return -BCH_ERR_ENOSPC_sb_members; +have_slot: + unsigned nr_devices = max_t(unsigned, dev_idx + 1, c->sb.nr_devices); + + struct bch_sb_field_members_v2 *mi = bch2_sb_field_get(c->disk_sb.sb, members_v2); + unsigned u64s = DIV_ROUND_UP(sizeof(struct bch_sb_field_members_v2) + + le16_to_cpu(mi->member_bytes) * nr_devices, sizeof(u64)); + + mi = bch2_sb_field_resize(&c->disk_sb, members_v2, u64s); + if (!mi) + return -BCH_ERR_ENOSPC_sb_members; + + c->disk_sb.sb->nr_devices = nr_devices; + return dev_idx; +} diff --git a/fs/bcachefs/sb-members.h b/fs/bcachefs/sb-members.h index f307f2857603..1a21d1f15a9c 100644 --- a/fs/bcachefs/sb-members.h +++ b/fs/bcachefs/sb-members.h @@ -354,4 +354,6 @@ static inline bool bch2_dev_btree_bitmap_marked_sectors(struct bch_dev *ca, u64 bool bch2_dev_btree_bitmap_marked(struct bch_fs *, struct bkey_s_c); void bch2_dev_btree_bitmap_mark(struct bch_fs *, struct bkey_s_c); +int bch2_sb_member_alloc(struct bch_fs *); + #endif /* _BCACHEFS_SB_MEMBERS_H */ diff --git a/fs/bcachefs/super.c b/fs/bcachefs/super.c index 7dba46e97fbd..1c949389d88f 100644 --- a/fs/bcachefs/super.c +++ b/fs/bcachefs/super.c @@ -1703,9 +1703,6 @@ int bch2_dev_add(struct bch_fs *c, const char *path) struct bch_opts opts = bch2_opts_empty(); struct bch_sb_handle sb; struct bch_dev *ca = NULL; - struct bch_sb_field_members_v2 *mi; - struct bch_member dev_mi; - unsigned dev_idx, nr_devices, u64s; struct printbuf errbuf = PRINTBUF; struct printbuf label = PRINTBUF; int ret; @@ -1715,7 +1712,7 @@ int bch2_dev_add(struct bch_fs *c, const char *path) if (ret) goto err; - dev_mi = bch2_sb_member_get(sb.sb, sb.sb->dev_idx); + struct bch_member dev_mi = bch2_sb_member_get(sb.sb, sb.sb->dev_idx); if (BCH_MEMBER_GROUP(&dev_mi)) { bch2_disk_path_to_text_sb(&label, sb.sb, BCH_MEMBER_GROUP(&dev_mi) - 1); @@ -1753,55 +1750,19 @@ int bch2_dev_add(struct bch_fs *c, const char *path) goto err_unlock; if (dynamic_fault("bcachefs:add:no_slot")) - goto no_slot; + goto err_unlock; - if (c->sb.nr_devices < BCH_SB_MEMBERS_MAX) { - dev_idx = c->sb.nr_devices; - goto have_slot; - } - - int best = -1; - u64 best_last_mount = 0; - for (dev_idx = 0; dev_idx < BCH_SB_MEMBERS_MAX; dev_idx++) { - struct bch_member m = bch2_sb_member_get(c->disk_sb.sb, dev_idx); - if (bch2_member_alive(&m)) - continue; - - u64 last_mount = le64_to_cpu(m.last_mount); - if (best < 0 || last_mount < best_last_mount) { - best = dev_idx; - best_last_mount = last_mount; - } - } - if (best >= 0) { - dev_idx = best; - goto have_slot; - } -no_slot: - ret = -BCH_ERR_ENOSPC_sb_members; - bch_err_msg(c, ret, "setting up new superblock"); - goto err_unlock; - -have_slot: - nr_devices = max_t(unsigned, dev_idx + 1, c->sb.nr_devices); - - mi = bch2_sb_field_get(c->disk_sb.sb, members_v2); - u64s = DIV_ROUND_UP(sizeof(struct bch_sb_field_members_v2) + - le16_to_cpu(mi->member_bytes) * nr_devices, sizeof(u64)); - - mi = bch2_sb_field_resize(&c->disk_sb, members_v2, u64s); - if (!mi) { - ret = -BCH_ERR_ENOSPC_sb_members; + ret = bch2_sb_member_alloc(c); + if (ret < 0) { bch_err_msg(c, ret, "setting up new superblock"); goto err_unlock; } - struct bch_member *m = bch2_members_v2_get_mut(c->disk_sb.sb, dev_idx); + unsigned dev_idx = ret; /* success: */ - *m = dev_mi; - m->last_mount = cpu_to_le64(ktime_get_real_seconds()); - c->disk_sb.sb->nr_devices = nr_devices; + dev_mi.last_mount = cpu_to_le64(ktime_get_real_seconds()); + *bch2_members_v2_get_mut(c->disk_sb.sb, dev_idx) = dev_mi; ca->disk_sb.sb->dev_idx = dev_idx; bch2_dev_attach(c, ca, dev_idx); From b161ca80968b55e551b82955e65264e8fc5f8660 Mon Sep 17 00:00:00 2001 From: Hongbo Li Date: Wed, 4 Sep 2024 15:15:32 +0800 Subject: [PATCH 279/302] bcachefs: Fix compilation error for bch2_sb_member_alloc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix the following compilation error: ``` fs/bcachefs/sb-members.c: In function ‘bch2_sb_member_alloc’: fs/bcachefs/sb-members.c:508:2: error: a label can only be part of a statement and a declaration is not a statement 508 | unsigned nr_devices = max_t(unsigned, dev_idx + 1, c->sb.nr_devices); ``` Fixes: a7d364a133c7 ("bcachefs: bch2_sb_member_alloc()") Signed-off-by: Hongbo Li Signed-off-by: Kent Overstreet --- fs/bcachefs/sb-members.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/fs/bcachefs/sb-members.c b/fs/bcachefs/sb-members.c index 2aad62961938..02bcde3c1b02 100644 --- a/fs/bcachefs/sb-members.c +++ b/fs/bcachefs/sb-members.c @@ -478,11 +478,15 @@ unsigned bch2_sb_nr_devices(const struct bch_sb *sb) int bch2_sb_member_alloc(struct bch_fs *c) { unsigned dev_idx = c->sb.nr_devices; + struct bch_sb_field_members_v2 *mi; + unsigned nr_devices; + unsigned u64s; + int best = -1; + u64 best_last_mount = 0; + if (dev_idx < BCH_SB_MEMBERS_MAX) goto have_slot; - int best = -1; - u64 best_last_mount = 0; for (dev_idx = 0; dev_idx < BCH_SB_MEMBERS_MAX; dev_idx++) { /* eventually BCH_SB_MEMBERS_MAX will be raised */ if (dev_idx == BCH_SB_MEMBER_INVALID) @@ -505,11 +509,11 @@ int bch2_sb_member_alloc(struct bch_fs *c) return -BCH_ERR_ENOSPC_sb_members; have_slot: - unsigned nr_devices = max_t(unsigned, dev_idx + 1, c->sb.nr_devices); + nr_devices = max_t(unsigned, dev_idx + 1, c->sb.nr_devices); - struct bch_sb_field_members_v2 *mi = bch2_sb_field_get(c->disk_sb.sb, members_v2); - unsigned u64s = DIV_ROUND_UP(sizeof(struct bch_sb_field_members_v2) + - le16_to_cpu(mi->member_bytes) * nr_devices, sizeof(u64)); + mi = bch2_sb_field_get(c->disk_sb.sb, members_v2); + u64s = DIV_ROUND_UP(sizeof(struct bch_sb_field_members_v2) + + le16_to_cpu(mi->member_bytes) * nr_devices, sizeof(u64)); mi = bch2_sb_field_resize(&c->disk_sb, members_v2, u64s); if (!mi) From bf611567b791f3dabf4197ce70021e3bf2ce69a5 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Fri, 6 Sep 2024 18:32:49 -0400 Subject: [PATCH 280/302] bcachefs: improve "no device to read from" message Signed-off-by: Kent Overstreet --- fs/bcachefs/io_read.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/fs/bcachefs/io_read.c b/fs/bcachefs/io_read.c index ce27ba1f0b54..10242671b269 100644 --- a/fs/bcachefs/io_read.c +++ b/fs/bcachefs/io_read.c @@ -869,9 +869,15 @@ retry_pick: goto hole; if (pick_ret < 0) { + struct printbuf buf = PRINTBUF; + bch2_bkey_val_to_text(&buf, c, k); + bch_err_inum_offset_ratelimited(c, read_pos.inode, read_pos.offset << 9, - "no device to read from"); + "no device to read from: %s\n %s", + bch2_err_str(pick_ret), + buf.buf); + printbuf_exit(&buf); goto err; } From 3621ecc10f831f4fd27784083dfaf5b8481098b5 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sat, 7 Sep 2024 20:27:23 -0400 Subject: [PATCH 281/302] bcachefs: bch2_opts_to_text() Factor out bch2_show_options() into a generic helper, for debugging option passing issues. Signed-off-by: Kent Overstreet --- fs/bcachefs/fs.c | 26 +++++--------------------- fs/bcachefs/opts.c | 26 ++++++++++++++++++++++++++ fs/bcachefs/opts.h | 4 ++++ 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/fs/bcachefs/fs.c b/fs/bcachefs/fs.c index dfadfb82399f..1aee5bafaae5 100644 --- a/fs/bcachefs/fs.c +++ b/fs/bcachefs/fs.c @@ -1930,30 +1930,14 @@ static int bch2_show_devname(struct seq_file *seq, struct dentry *root) static int bch2_show_options(struct seq_file *seq, struct dentry *root) { struct bch_fs *c = root->d_sb->s_fs_info; - enum bch_opt_id i; struct printbuf buf = PRINTBUF; - int ret = 0; - for (i = 0; i < bch2_opts_nr; i++) { - const struct bch_option *opt = &bch2_opt_table[i]; - u64 v = bch2_opt_get_by_id(&c->opts, i); + bch2_opts_to_text(&buf, c->opts, c, c->disk_sb.sb, + OPT_MOUNT, OPT_HIDDEN, OPT_SHOW_MOUNT_STYLE); + printbuf_nul_terminate(&buf); + seq_puts(seq, buf.buf); - if ((opt->flags & OPT_HIDDEN) || - !(opt->flags & OPT_MOUNT)) - continue; - - if (v == bch2_opt_get_by_id(&bch2_opts_default, i)) - continue; - - printbuf_reset(&buf); - bch2_opt_to_text(&buf, c, c->disk_sb.sb, opt, v, - OPT_SHOW_MOUNT_STYLE); - seq_putc(seq, ','); - seq_puts(seq, buf.buf); - } - - if (buf.allocation_failure) - ret = -ENOMEM; + int ret = buf.allocation_failure ? -ENOMEM : 0; printbuf_exit(&buf); return ret; } diff --git a/fs/bcachefs/opts.c b/fs/bcachefs/opts.c index 9cb9b05db6d1..232be8a44051 100644 --- a/fs/bcachefs/opts.c +++ b/fs/bcachefs/opts.c @@ -443,6 +443,32 @@ void bch2_opt_to_text(struct printbuf *out, } } +void bch2_opts_to_text(struct printbuf *out, + struct bch_opts opts, + struct bch_fs *c, struct bch_sb *sb, + unsigned show_mask, unsigned hide_mask, + unsigned flags) +{ + bool first = true; + + for (enum bch_opt_id i = 0; i < bch2_opts_nr; i++) { + const struct bch_option *opt = &bch2_opt_table[i]; + + if ((opt->flags & hide_mask) || !(opt->flags & show_mask)) + continue; + + u64 v = bch2_opt_get_by_id(&opts, i); + if (v == bch2_opt_get_by_id(&bch2_opts_default, i)) + continue; + + if (!first) + prt_char(out, ','); + first = false; + + bch2_opt_to_text(out, c, sb, opt, v, flags); + } +} + int bch2_opt_check_may_set(struct bch_fs *c, int id, u64 v) { int ret = 0; diff --git a/fs/bcachefs/opts.h b/fs/bcachefs/opts.h index f2ae13f0fbad..cb2e244a2429 100644 --- a/fs/bcachefs/opts.h +++ b/fs/bcachefs/opts.h @@ -605,6 +605,10 @@ int bch2_opt_parse(struct bch_fs *, const struct bch_option *, void bch2_opt_to_text(struct printbuf *, struct bch_fs *, struct bch_sb *, const struct bch_option *, u64, unsigned); +void bch2_opts_to_text(struct printbuf *, + struct bch_opts, + struct bch_fs *, struct bch_sb *, + unsigned, unsigned, unsigned); int bch2_opt_check_may_set(struct bch_fs *, int, u64); int bch2_opts_check_may_set(struct bch_fs *); From b99a94fd7ae91951d4f96b39e6ff50ac2fa8decb Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 27 Aug 2024 20:21:03 -0400 Subject: [PATCH 282/302] bcachefs: Progress indicator for extents_to_backpointers Signed-off-by: Kent Overstreet --- fs/bcachefs/backpointers.c | 86 +++++++++++++++++++++++++++++++++++--- 1 file changed, 81 insertions(+), 5 deletions(-) diff --git a/fs/bcachefs/backpointers.c b/fs/bcachefs/backpointers.c index d4da6343efa9..2bdce3185f29 100644 --- a/fs/bcachefs/backpointers.c +++ b/fs/bcachefs/backpointers.c @@ -3,12 +3,14 @@ #include "bbpos.h" #include "alloc_background.h" #include "backpointers.h" +#include "bbpos.h" #include "bkey_buf.h" #include "btree_cache.h" #include "btree_update.h" #include "btree_update_interior.h" #include "btree_write_buffer.h" #include "checksum.h" +#include "disk_accounting.h" #include "error.h" #include @@ -782,12 +784,80 @@ static int bch2_get_btree_in_memory_pos(struct btree_trans *trans, return ret; } +struct progress_indicator_state { + unsigned long next_print; + u64 nodes_seen; + u64 nodes_total; + struct btree *last_node; +}; + +static inline void progress_init(struct progress_indicator_state *s, + struct bch_fs *c, + u64 btree_id_mask) +{ + memset(s, 0, sizeof(*s)); + + s->next_print = jiffies + HZ * 10; + + for (unsigned i = 0; i < BTREE_ID_NR; i++) { + if (!(btree_id_mask & BIT_ULL(i))) + continue; + + struct disk_accounting_pos acc = { + .type = BCH_DISK_ACCOUNTING_btree, + .btree.id = i, + }; + + u64 v; + bch2_accounting_mem_read(c, disk_accounting_pos_to_bpos(&acc), &v, 1); + s->nodes_total += div64_ul(v, btree_sectors(c)); + } +} + +static inline bool progress_update_p(struct progress_indicator_state *s) +{ + bool ret = time_after_eq(jiffies, s->next_print); + + if (ret) + s->next_print = jiffies + HZ * 10; + return ret; +} + +static void progress_update_iter(struct btree_trans *trans, + struct progress_indicator_state *s, + struct btree_iter *iter, + const char *msg) +{ + struct bch_fs *c = trans->c; + struct btree *b = path_l(btree_iter_path(trans, iter))->b; + + s->nodes_seen += b != s->last_node; + s->last_node = b; + + if (progress_update_p(s)) { + struct printbuf buf = PRINTBUF; + unsigned percent = s->nodes_total + ? div64_u64(s->nodes_seen * 100, s->nodes_total) + : 0; + + prt_printf(&buf, "%s: %d%%, done %llu/%llu nodes, at ", + msg, percent, s->nodes_seen, s->nodes_total); + bch2_bbpos_to_text(&buf, BBPOS(iter->btree_id, iter->pos)); + + bch_info(c, "%s", buf.buf); + printbuf_exit(&buf); + } +} + static int bch2_check_extents_to_backpointers_pass(struct btree_trans *trans, struct extents_to_bp_state *s) { struct bch_fs *c = trans->c; + struct progress_indicator_state progress; int ret = 0; + progress_init(&progress, trans->c, BIT_ULL(BTREE_ID_extents)|BIT_ULL(BTREE_ID_reflink)); + for (enum btree_id btree_id = 0; btree_id < btree_id_nr_alive(c); btree_id++) { @@ -805,6 +875,7 @@ static int bch2_check_extents_to_backpointers_pass(struct btree_trans *trans, BTREE_ITER_prefetch); ret = for_each_btree_key_continue(trans, iter, 0, k, ({ + progress_update_iter(trans, &progress, &iter, "extents_to_backpointers"); check_extent_to_backpointers(trans, s, btree_id, level, k) ?: bch2_trans_commit(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc); })); @@ -920,19 +991,24 @@ static int bch2_check_backpointers_to_extents_pass(struct btree_trans *trans, struct bbpos start, struct bbpos end) { + struct bch_fs *c = trans->c; struct bkey_buf last_flushed; + struct progress_indicator_state progress; bch2_bkey_buf_init(&last_flushed); bkey_init(&last_flushed.k->k); + progress_init(&progress, trans->c, BIT_ULL(BTREE_ID_backpointers)); int ret = for_each_btree_key_commit(trans, iter, BTREE_ID_backpointers, POS_MIN, BTREE_ITER_prefetch, k, - NULL, NULL, BCH_TRANS_COMMIT_no_enospc, - check_one_backpointer(trans, start, end, - bkey_s_c_to_backpointer(k), - &last_flushed)); + NULL, NULL, BCH_TRANS_COMMIT_no_enospc, ({ + progress_update_iter(trans, &progress, &iter, "backpointers_to_extents"); + check_one_backpointer(trans, start, end, + bkey_s_c_to_backpointer(k), + &last_flushed); + })); - bch2_bkey_buf_exit(&last_flushed, trans->c); + bch2_bkey_buf_exit(&last_flushed, c); return ret; } From 805ddc204287e74ebd2640c0f088e04f94e98b48 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 1 Sep 2024 18:12:26 -0400 Subject: [PATCH 283/302] bcachefs: bch2_dev_rcu_noerror() bch2_dev_rcu() now properly errors if the device is invalid Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_background.h | 2 +- fs/bcachefs/backpointers.c | 6 +++--- fs/bcachefs/buckets.c | 2 +- fs/bcachefs/extents.c | 5 +++-- fs/bcachefs/replicas.c | 2 +- fs/bcachefs/sb-members.h | 18 +++++++++++++----- 6 files changed, 22 insertions(+), 13 deletions(-) diff --git a/fs/bcachefs/alloc_background.h b/fs/bcachefs/alloc_background.h index 577f823a0207..f8e87c6721b1 100644 --- a/fs/bcachefs/alloc_background.h +++ b/fs/bcachefs/alloc_background.h @@ -16,7 +16,7 @@ enum bch_validate_flags; static inline bool bch2_dev_bucket_exists(struct bch_fs *c, struct bpos pos) { rcu_read_lock(); - struct bch_dev *ca = bch2_dev_rcu(c, pos.inode); + struct bch_dev *ca = bch2_dev_rcu_noerror(c, pos.inode); bool ret = ca && bucket_valid(ca, pos.offset); rcu_read_unlock(); return ret; diff --git a/fs/bcachefs/backpointers.c b/fs/bcachefs/backpointers.c index 2bdce3185f29..6c395445f60c 100644 --- a/fs/bcachefs/backpointers.c +++ b/fs/bcachefs/backpointers.c @@ -55,7 +55,7 @@ int bch2_backpointer_validate(struct bch_fs *c, struct bkey_s_c k, struct bkey_s_c_backpointer bp = bkey_s_c_to_backpointer(k); rcu_read_lock(); - struct bch_dev *ca = bch2_dev_rcu(c, bp.k->p.inode); + struct bch_dev *ca = bch2_dev_rcu_noerror(c, bp.k->p.inode); if (!ca) { /* these will be caught by fsck */ rcu_read_unlock(); @@ -89,7 +89,7 @@ void bch2_backpointer_to_text(struct printbuf *out, const struct bch_backpointer void bch2_backpointer_k_to_text(struct printbuf *out, struct bch_fs *c, struct bkey_s_c k) { rcu_read_lock(); - struct bch_dev *ca = bch2_dev_rcu(c, k.k->p.inode); + struct bch_dev *ca = bch2_dev_rcu_noerror(c, k.k->p.inode); if (ca) { struct bpos bucket = bp_pos_to_bucket(ca, k.k->p); rcu_read_unlock(); @@ -673,7 +673,7 @@ static int check_extent_to_backpointers(struct btree_trans *trans, continue; rcu_read_lock(); - struct bch_dev *ca = bch2_dev_rcu(c, p.ptr.dev); + struct bch_dev *ca = bch2_dev_rcu_noerror(c, p.ptr.dev); if (ca) bch2_extent_ptr_to_bp(c, ca, btree, level, k, p, entry, &bucket_pos, &bp); rcu_read_unlock(); diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c index 4abd22d7e3f8..2d1d4dae631b 100644 --- a/fs/bcachefs/buckets.c +++ b/fs/bcachefs/buckets.c @@ -281,7 +281,7 @@ int bch2_check_fix_ptrs(struct btree_trans *trans, goto err; rcu_read_lock(); - bch2_bkey_drop_ptrs(bkey_i_to_s(new), ptr, !bch2_dev_rcu(c, ptr->dev)); + bch2_bkey_drop_ptrs(bkey_i_to_s(new), ptr, !bch2_dev_exists(c, ptr->dev)); rcu_read_unlock(); if (level) { diff --git a/fs/bcachefs/extents.c b/fs/bcachefs/extents.c index 324303bf4353..ad63445b85e5 100644 --- a/fs/bcachefs/extents.c +++ b/fs/bcachefs/extents.c @@ -1021,7 +1021,7 @@ void bch2_extent_ptr_to_text(struct printbuf *out, struct bch_fs *c, const struc { out->atomic++; rcu_read_lock(); - struct bch_dev *ca = bch2_dev_rcu(c, ptr->dev); + struct bch_dev *ca = bch2_dev_rcu_noerror(c, ptr->dev); if (!ca) { prt_printf(out, "ptr: %u:%llu gen %u%s", ptr->dev, (u64) ptr->offset, ptr->gen, @@ -1125,8 +1125,9 @@ static int extent_ptr_validate(struct bch_fs *c, { int ret = 0; + /* bad pointers are repaired by check_fix_ptrs(): */ rcu_read_lock(); - struct bch_dev *ca = bch2_dev_rcu(c, ptr->dev); + struct bch_dev *ca = bch2_dev_rcu_noerror(c, ptr->dev); if (!ca) { rcu_read_unlock(); return 0; diff --git a/fs/bcachefs/replicas.c b/fs/bcachefs/replicas.c index 659ee0516c25..226b41baea2f 100644 --- a/fs/bcachefs/replicas.c +++ b/fs/bcachefs/replicas.c @@ -795,7 +795,7 @@ bool bch2_have_enough_devs(struct bch_fs *c, struct bch_devs_mask devs, for (unsigned i = 0; i < e->nr_devs; i++) { nr_online += test_bit(e->devs[i], devs.d); - struct bch_dev *ca = bch2_dev_rcu(c, e->devs[i]); + struct bch_dev *ca = bch2_dev_rcu_noerror(c, e->devs[i]); nr_failed += !ca || ca->mi.state == BCH_MEMBER_STATE_failed; } rcu_read_unlock(); diff --git a/fs/bcachefs/sb-members.h b/fs/bcachefs/sb-members.h index 1a21d1f15a9c..762083b564ee 100644 --- a/fs/bcachefs/sb-members.h +++ b/fs/bcachefs/sb-members.h @@ -198,29 +198,37 @@ static inline struct bch_dev *bch2_dev_locked(struct bch_fs *c, unsigned dev) lockdep_is_held(&c->state_lock)); } -static inline struct bch_dev *bch2_dev_rcu(struct bch_fs *c, unsigned dev) +static inline struct bch_dev *bch2_dev_rcu_noerror(struct bch_fs *c, unsigned dev) { return c && dev < c->sb.nr_devices ? rcu_dereference(c->devs[dev]) : NULL; } +void bch2_dev_missing(struct bch_fs *, unsigned); + +static inline struct bch_dev *bch2_dev_rcu(struct bch_fs *c, unsigned dev) +{ + struct bch_dev *ca = bch2_dev_rcu_noerror(c, dev); + if (unlikely(!ca)) + bch2_dev_missing(c, dev); + return ca; +} + static inline struct bch_dev *bch2_dev_tryget_noerror(struct bch_fs *c, unsigned dev) { rcu_read_lock(); - struct bch_dev *ca = bch2_dev_rcu(c, dev); + struct bch_dev *ca = bch2_dev_rcu_noerror(c, dev); if (ca) bch2_dev_get(ca); rcu_read_unlock(); return ca; } -void bch2_dev_missing(struct bch_fs *, unsigned); - static inline struct bch_dev *bch2_dev_tryget(struct bch_fs *c, unsigned dev) { struct bch_dev *ca = bch2_dev_tryget_noerror(c, dev); - if (!ca) + if (unlikely(!ca)) bch2_dev_missing(c, dev); return ca; } From e92e5056e4f75c1990d54ea5bc03360fca55b891 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sat, 7 Sep 2024 11:45:21 -0400 Subject: [PATCH 284/302] bcachefs: Failed devices no longer require mounting in degraded mode Signed-off-by: Kent Overstreet --- fs/bcachefs/replicas.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/bcachefs/replicas.c b/fs/bcachefs/replicas.c index 226b41baea2f..998c0bd06802 100644 --- a/fs/bcachefs/replicas.c +++ b/fs/bcachefs/replicas.c @@ -800,7 +800,7 @@ bool bch2_have_enough_devs(struct bch_fs *c, struct bch_devs_mask devs, } rcu_read_unlock(); - if (nr_failed == e->nr_devs) + if (nr_online + nr_failed == e->nr_devs) continue; if (nr_online < e->nr_required) From ad5dbe3ce533ec13abacad78076050672e3d39eb Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 4 Sep 2024 17:19:24 -0400 Subject: [PATCH 285/302] bcachefs: Don't count "skipped access bit" as touched in btree cache scan Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_cache.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/bcachefs/btree_cache.c b/fs/bcachefs/btree_cache.c index 9a2e6c6ea82c..585301822692 100644 --- a/fs/bcachefs/btree_cache.c +++ b/fs/bcachefs/btree_cache.c @@ -449,6 +449,7 @@ restart: if (btree_node_accessed(b)) { clear_btree_node_accessed(b); bc->not_freed[BCH_BTREE_CACHE_NOT_FREED_access_bit]++; + --touched;; } else if (!btree_node_reclaim(c, b, true)) { freed++; btree_node_data_free(c, b); From 691f2cba229189033c55f19b904bc6f4bd68b480 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 5 Sep 2024 19:25:01 -0400 Subject: [PATCH 286/302] bcachefs: btree cache counters should be size_t 32 bits won't overflow any time soon, but size_t is the correct type for counting objects in memory. Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_cache.c | 46 +++++++++++++++++------------------ fs/bcachefs/btree_io.c | 2 +- fs/bcachefs/btree_io.h | 4 +-- fs/bcachefs/btree_types.h | 13 +++++----- fs/bcachefs/journal_reclaim.c | 6 ++--- fs/bcachefs/super.c | 2 +- 6 files changed, 37 insertions(+), 36 deletions(-) diff --git a/fs/bcachefs/btree_cache.c b/fs/bcachefs/btree_cache.c index 585301822692..a7eb07d6e7f9 100644 --- a/fs/bcachefs/btree_cache.c +++ b/fs/bcachefs/btree_cache.c @@ -32,24 +32,24 @@ const char * const bch2_btree_node_flags[] = { void bch2_recalc_btree_reserve(struct bch_fs *c) { - unsigned i, reserve = 16; + unsigned reserve = 16; if (!c->btree_roots_known[0].b) reserve += 8; - for (i = 0; i < btree_id_nr_alive(c); i++) { + for (unsigned i = 0; i < btree_id_nr_alive(c); i++) { struct btree_root *r = bch2_btree_id_root(c, i); if (r->b) reserve += min_t(unsigned, 1, r->b->c.level) * 8; } - c->btree_cache.reserve = reserve; + c->btree_cache.nr_reserve = reserve; } -static inline unsigned btree_cache_can_free(struct btree_cache *bc) +static inline size_t btree_cache_can_free(struct btree_cache *bc) { - return max_t(int, 0, bc->used - bc->reserve); + return max_t(int, 0, bc->nr_used - bc->nr_reserve); } static void btree_node_to_freedlist(struct btree_cache *bc, struct btree *b) @@ -87,7 +87,7 @@ static void btree_node_data_free(struct bch_fs *c, struct btree *b) #endif b->aux_data = NULL; - bc->used--; + bc->nr_used--; btree_node_to_freedlist(bc, b); } @@ -167,7 +167,7 @@ struct btree *__bch2_btree_node_mem_alloc(struct bch_fs *c) bch2_btree_lock_init(&b->c, 0); - bc->used++; + bc->nr_used++; list_add(&b->list, &bc->freeable); return b; } @@ -194,7 +194,7 @@ void bch2_btree_node_hash_remove(struct btree_cache *bc, struct btree *b) b->hash_val = 0; if (b->c.btree_id < BTREE_ID_NR) - --bc->used_by_btree[b->c.btree_id]; + --bc->nr_by_btree[b->c.btree_id]; } int __bch2_btree_node_hash_insert(struct btree_cache *bc, struct btree *b) @@ -205,7 +205,7 @@ int __bch2_btree_node_hash_insert(struct btree_cache *bc, struct btree *b) int ret = rhashtable_lookup_insert_fast(&bc->table, &b->hash, bch_btree_cache_params); if (!ret && b->c.btree_id < BTREE_ID_NR) - bc->used_by_btree[b->c.btree_id]++; + bc->nr_by_btree[b->c.btree_id]++; return ret; } @@ -401,8 +401,8 @@ static unsigned long bch2_btree_cache_scan(struct shrinker *shrink, unsigned long touched = 0; unsigned i, flags; unsigned long ret = SHRINK_STOP; - bool trigger_writes = atomic_read(&bc->dirty) + nr >= - bc->used * 3 / 4; + bool trigger_writes = atomic_long_read(&bc->nr_dirty) + nr >= + bc->nr_used * 3 / 4; if (bch2_btree_shrinker_disabled) return SHRINK_STOP; @@ -439,7 +439,7 @@ static unsigned long bch2_btree_cache_scan(struct shrinker *shrink, six_unlock_write(&b->c.lock); six_unlock_intent(&b->c.lock); freed++; - bc->freed++; + bc->nr_freed++; } } restart: @@ -453,7 +453,7 @@ restart: } else if (!btree_node_reclaim(c, b, true)) { freed++; btree_node_data_free(c, b); - bc->freed++; + bc->nr_freed++; bch2_btree_node_hash_remove(bc, b); six_unlock_write(&b->c.lock); @@ -539,7 +539,7 @@ void bch2_fs_btree_cache_exit(struct bch_fs *c) } BUG_ON(!bch2_journal_error(&c->journal) && - atomic_read(&c->btree_cache.dirty)); + atomic_long_read(&c->btree_cache.nr_dirty)); list_splice(&bc->freed_pcpu, &bc->freed_nonpcpu); @@ -572,7 +572,7 @@ int bch2_fs_btree_cache_init(struct bch_fs *c) bch2_recalc_btree_reserve(c); - for (i = 0; i < bc->reserve; i++) + for (i = 0; i < bc->nr_reserve; i++) if (!__bch2_btree_node_mem_alloc(c)) goto err; @@ -739,7 +739,7 @@ got_node: } mutex_lock(&bc->lock); - bc->used++; + bc->nr_used++; got_mem: mutex_unlock(&bc->lock); @@ -1353,11 +1353,11 @@ void bch2_btree_node_to_text(struct printbuf *out, struct bch_fs *c, const struc } static void prt_btree_cache_line(struct printbuf *out, const struct bch_fs *c, - const char *label, unsigned nr) + const char *label, size_t nr) { prt_printf(out, "%s\t", label); prt_human_readable_u64(out, nr * c->opts.btree_node_size); - prt_printf(out, " (%u)\n", nr); + prt_printf(out, " (%zu)\n", nr); } static const char * const bch2_btree_cache_not_freed_reasons_strs[] = { @@ -1374,16 +1374,16 @@ void bch2_btree_cache_to_text(struct printbuf *out, const struct btree_cache *bc if (!out->nr_tabstops) printbuf_tabstop_push(out, 32); - prt_btree_cache_line(out, c, "total:", bc->used); - prt_btree_cache_line(out, c, "nr dirty:", atomic_read(&bc->dirty)); + prt_btree_cache_line(out, c, "total:", bc->nr_used); + prt_btree_cache_line(out, c, "nr dirty:", atomic_long_read(&bc->nr_dirty)); prt_printf(out, "cannibalize lock:\t%p\n", bc->alloc_lock); prt_newline(out); - for (unsigned i = 0; i < ARRAY_SIZE(bc->used_by_btree); i++) - prt_btree_cache_line(out, c, bch2_btree_id_str(i), bc->used_by_btree[i]); + for (unsigned i = 0; i < ARRAY_SIZE(bc->nr_by_btree); i++) + prt_btree_cache_line(out, c, bch2_btree_id_str(i), bc->nr_by_btree[i]); prt_newline(out); - prt_printf(out, "freed:\t%u\n", bc->freed); + prt_printf(out, "freed:\t%zu\n", bc->nr_freed); prt_printf(out, "not freed:\n"); for (unsigned i = 0; i < ARRAY_SIZE(bc->not_freed); i++) diff --git a/fs/bcachefs/btree_io.c b/fs/bcachefs/btree_io.c index 18cff98650de..aad89ba16b9b 100644 --- a/fs/bcachefs/btree_io.c +++ b/fs/bcachefs/btree_io.c @@ -2031,7 +2031,7 @@ void __bch2_btree_node_write(struct bch_fs *c, struct btree *b, unsigned flags) do_write: BUG_ON((type == BTREE_WRITE_initial) != (b->written == 0)); - atomic_dec(&c->btree_cache.dirty); + atomic_long_dec(&c->btree_cache.nr_dirty); BUG_ON(btree_node_fake(b)); BUG_ON((b->will_make_reachable != 0) != !b->written); diff --git a/fs/bcachefs/btree_io.h b/fs/bcachefs/btree_io.h index 63d76f5c6403..9b01ca3de907 100644 --- a/fs/bcachefs/btree_io.h +++ b/fs/bcachefs/btree_io.h @@ -18,13 +18,13 @@ struct btree_node_read_all; static inline void set_btree_node_dirty_acct(struct bch_fs *c, struct btree *b) { if (!test_and_set_bit(BTREE_NODE_dirty, &b->flags)) - atomic_inc(&c->btree_cache.dirty); + atomic_long_inc(&c->btree_cache.nr_dirty); } static inline void clear_btree_node_dirty_acct(struct bch_fs *c, struct btree *b) { if (test_and_clear_bit(BTREE_NODE_dirty, &b->flags)) - atomic_dec(&c->btree_cache.dirty); + atomic_long_dec(&c->btree_cache.nr_dirty); } static inline unsigned btree_ptr_sectors_written(struct bkey_s_c k) diff --git a/fs/bcachefs/btree_types.h b/fs/bcachefs/btree_types.h index c1ab824e1c34..806d27b7f41b 100644 --- a/fs/bcachefs/btree_types.h +++ b/fs/bcachefs/btree_types.h @@ -180,15 +180,16 @@ struct btree_cache { struct list_head freed_nonpcpu; /* Number of elements in live + freeable lists */ - unsigned used; - unsigned reserve; - unsigned freed; - atomic_t dirty; + size_t nr_used; + size_t nr_reserve; + size_t nr_by_btree[BTREE_ID_NR]; + atomic_long_t nr_dirty; + + /* shrinker stats */ + size_t nr_freed; u64 not_freed[BCH_BTREE_CACHE_NOT_FREED_REASONS_NR]; struct shrinker *shrink; - unsigned used_by_btree[BTREE_ID_NR]; - /* * If we need to allocate memory for a new btree node and that * allocation fails, we can cannibalize another node in the btree cache diff --git a/fs/bcachefs/journal_reclaim.c b/fs/bcachefs/journal_reclaim.c index 70b998d9f19c..9794b6d214cd 100644 --- a/fs/bcachefs/journal_reclaim.c +++ b/fs/bcachefs/journal_reclaim.c @@ -681,7 +681,7 @@ static int __bch2_journal_reclaim(struct journal *j, bool direct, bool kicked) if (j->watermark != BCH_WATERMARK_stripe) min_nr = 1; - if (atomic_read(&c->btree_cache.dirty) * 2 > c->btree_cache.used) + if (atomic_long_read(&c->btree_cache.nr_dirty) * 2 > c->btree_cache.nr_used) min_nr = 1; min_key_cache = min(bch2_nr_btree_keys_need_flush(c), (size_t) 128); @@ -689,8 +689,8 @@ static int __bch2_journal_reclaim(struct journal *j, bool direct, bool kicked) trace_and_count(c, journal_reclaim_start, c, direct, kicked, min_nr, min_key_cache, - atomic_read(&c->btree_cache.dirty), - c->btree_cache.used, + atomic_long_read(&c->btree_cache.nr_dirty), + c->btree_cache.nr_used, atomic_long_read(&c->btree_key_cache.nr_dirty), atomic_long_read(&c->btree_key_cache.nr_keys)); diff --git a/fs/bcachefs/super.c b/fs/bcachefs/super.c index 1c949389d88f..873e4be7e1dc 100644 --- a/fs/bcachefs/super.c +++ b/fs/bcachefs/super.c @@ -370,7 +370,7 @@ void bch2_fs_read_only(struct bch_fs *c) test_bit(BCH_FS_clean_shutdown, &c->flags) && c->recovery_pass_done >= BCH_RECOVERY_PASS_journal_replay) { BUG_ON(c->journal.last_empty_seq != journal_cur_seq(&c->journal)); - BUG_ON(atomic_read(&c->btree_cache.dirty)); + BUG_ON(atomic_long_read(&c->btree_cache.nr_dirty)); BUG_ON(atomic_long_read(&c->btree_key_cache.nr_dirty)); BUG_ON(c->btree_write_buffer.inc.keys.nr); BUG_ON(c->btree_write_buffer.flushing.keys.nr); From 91ddd7151000c0e538cec7fb2f3f86e2268af4d4 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 5 Sep 2024 19:37:56 -0400 Subject: [PATCH 287/302] bcachefs: split up btree cache counters for live, freeable this is prep for introducing a second live list and shrinker for pinned nodes Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_cache.c | 58 ++++++++++++++++++----------- fs/bcachefs/btree_gc.c | 3 +- fs/bcachefs/btree_io.c | 4 +- fs/bcachefs/btree_types.h | 4 +- fs/bcachefs/btree_update_interior.c | 6 ++- fs/bcachefs/journal_reclaim.c | 4 +- 6 files changed, 47 insertions(+), 32 deletions(-) diff --git a/fs/bcachefs/btree_cache.c b/fs/bcachefs/btree_cache.c index a7eb07d6e7f9..e66853b98857 100644 --- a/fs/bcachefs/btree_cache.c +++ b/fs/bcachefs/btree_cache.c @@ -49,7 +49,7 @@ void bch2_recalc_btree_reserve(struct bch_fs *c) static inline size_t btree_cache_can_free(struct btree_cache *bc) { - return max_t(int, 0, bc->nr_used - bc->nr_reserve); + return max_t(int, 0, bc->nr_live + bc->nr_freeable - bc->nr_reserve); } static void btree_node_to_freedlist(struct btree_cache *bc, struct btree *b) @@ -64,6 +64,8 @@ static void btree_node_data_free(struct bch_fs *c, struct btree *b) { struct btree_cache *bc = &c->btree_cache; + BUG_ON(btree_node_hashed(b)); + /* * This should really be done in slub/vmalloc, but we're using the * kmalloc_large() path, so we're working around a slub bug by doing @@ -87,7 +89,7 @@ static void btree_node_data_free(struct bch_fs *c, struct btree *b) #endif b->aux_data = NULL; - bc->nr_used--; + bc->nr_freeable--; btree_node_to_freedlist(bc, b); } @@ -167,7 +169,7 @@ struct btree *__bch2_btree_node_mem_alloc(struct bch_fs *c) bch2_btree_lock_init(&b->c, 0); - bc->nr_used++; + bc->nr_freeable++; list_add(&b->list, &bc->freeable); return b; } @@ -186,6 +188,7 @@ void bch2_btree_node_to_freelist(struct bch_fs *c, struct btree *b) void bch2_btree_node_hash_remove(struct btree_cache *bc, struct btree *b) { + lockdep_assert_held(&bc->lock); int ret = rhashtable_remove_fast(&bc->table, &b->hash, bch_btree_cache_params); BUG_ON(ret); @@ -195,6 +198,10 @@ void bch2_btree_node_hash_remove(struct btree_cache *bc, struct btree *b) if (b->c.btree_id < BTREE_ID_NR) --bc->nr_by_btree[b->c.btree_id]; + + bc->nr_live--; + bc->nr_freeable++; + list_move(&b->list, &bc->freeable); } int __bch2_btree_node_hash_insert(struct btree_cache *bc, struct btree *b) @@ -204,23 +211,25 @@ int __bch2_btree_node_hash_insert(struct btree_cache *bc, struct btree *b) int ret = rhashtable_lookup_insert_fast(&bc->table, &b->hash, bch_btree_cache_params); - if (!ret && b->c.btree_id < BTREE_ID_NR) + if (ret) + return ret; + + if (b->c.btree_id < BTREE_ID_NR) bc->nr_by_btree[b->c.btree_id]++; - return ret; + bc->nr_live++; + bc->nr_freeable--; + list_move_tail(&b->list, &bc->live); + return 0; } int bch2_btree_node_hash_insert(struct btree_cache *bc, struct btree *b, unsigned level, enum btree_id id) { - int ret; - b->c.level = level; b->c.btree_id = id; mutex_lock(&bc->lock); - ret = __bch2_btree_node_hash_insert(bc, b); - if (!ret) - list_add_tail(&b->list, &bc->live); + int ret = __bch2_btree_node_hash_insert(bc, b); mutex_unlock(&bc->lock); return ret; @@ -402,7 +411,7 @@ static unsigned long bch2_btree_cache_scan(struct shrinker *shrink, unsigned i, flags; unsigned long ret = SHRINK_STOP; bool trigger_writes = atomic_long_read(&bc->nr_dirty) + nr >= - bc->nr_used * 3 / 4; + (bc->nr_live + bc->nr_freeable) * 3 / 4; if (bch2_btree_shrinker_disabled) return SHRINK_STOP; @@ -451,11 +460,12 @@ restart: bc->not_freed[BCH_BTREE_CACHE_NOT_FREED_access_bit]++; --touched;; } else if (!btree_node_reclaim(c, b, true)) { + bch2_btree_node_hash_remove(bc, b); + freed++; btree_node_data_free(c, b); bc->nr_freed++; - bch2_btree_node_hash_remove(bc, b); six_unlock_write(&b->c.lock); six_unlock_intent(&b->c.lock); @@ -506,7 +516,7 @@ static unsigned long bch2_btree_cache_count(struct shrinker *shrink, void bch2_fs_btree_cache_exit(struct bch_fs *c) { struct btree_cache *bc = &c->btree_cache; - struct btree *b; + struct btree *b, *t; unsigned i, flags; shrinker_free(bc->shrink); @@ -527,11 +537,10 @@ void bch2_fs_btree_cache_exit(struct bch_fs *c) list_add(&r->b->list, &bc->live); } - list_splice(&bc->freeable, &bc->live); - - while (!list_empty(&bc->live)) { - b = list_first_entry(&bc->live, struct btree, list); + list_for_each_entry_safe(b, t, &bc->live, list) + bch2_btree_node_hash_remove(bc, b); + list_for_each_entry_safe(b, t, &bc->freeable, list) { BUG_ON(btree_node_read_in_flight(b) || btree_node_write_in_flight(b)); @@ -543,8 +552,7 @@ void bch2_fs_btree_cache_exit(struct bch_fs *c) list_splice(&bc->freed_pcpu, &bc->freed_nonpcpu); - while (!list_empty(&bc->freed_nonpcpu)) { - b = list_first_entry(&bc->freed_nonpcpu, struct btree, list); + list_for_each_entry_safe(b, t, &bc->freed_nonpcpu, list) { list_del(&b->list); six_lock_exit(&b->c.lock); kfree(b); @@ -553,6 +561,11 @@ void bch2_fs_btree_cache_exit(struct bch_fs *c) mutex_unlock(&bc->lock); memalloc_nofs_restore(flags); + for (unsigned i = 0; i < ARRAY_SIZE(bc->nr_by_btree); i++) + BUG_ON(bc->nr_by_btree[i]); + BUG_ON(bc->nr_live); + BUG_ON(bc->nr_freeable); + if (bc->table_init_done) rhashtable_destroy(&bc->table); } @@ -739,7 +752,7 @@ got_node: } mutex_lock(&bc->lock); - bc->nr_used++; + bc->nr_freeable++; got_mem: mutex_unlock(&bc->lock); @@ -1280,8 +1293,8 @@ wait_on_io: BUG_ON(btree_node_dirty(b)); mutex_lock(&bc->lock); - btree_node_data_free(c, b); bch2_btree_node_hash_remove(bc, b); + btree_node_data_free(c, b); mutex_unlock(&bc->lock); out: six_unlock_write(&b->c.lock); @@ -1374,7 +1387,8 @@ void bch2_btree_cache_to_text(struct printbuf *out, const struct btree_cache *bc if (!out->nr_tabstops) printbuf_tabstop_push(out, 32); - prt_btree_cache_line(out, c, "total:", bc->nr_used); + prt_btree_cache_line(out, c, "nr_live:", bc->nr_live); + prt_btree_cache_line(out, c, "nr_freeable:", bc->nr_freeable); prt_btree_cache_line(out, c, "nr dirty:", atomic_long_read(&bc->nr_dirty)); prt_printf(out, "cannibalize lock:\t%p\n", bc->alloc_lock); prt_newline(out); diff --git a/fs/bcachefs/btree_gc.c b/fs/bcachefs/btree_gc.c index 120ffd68ab0a..b5e0692f03c6 100644 --- a/fs/bcachefs/btree_gc.c +++ b/fs/bcachefs/btree_gc.c @@ -549,9 +549,8 @@ reconstruct_root: six_unlock_read(&b->c.lock); if (ret == DROP_THIS_NODE) { - bch2_btree_node_hash_remove(&c->btree_cache, b); mutex_lock(&c->btree_cache.lock); - list_move(&b->list, &c->btree_cache.freeable); + bch2_btree_node_hash_remove(&c->btree_cache, b); mutex_unlock(&c->btree_cache.lock); r->b = NULL; diff --git a/fs/bcachefs/btree_io.c b/fs/bcachefs/btree_io.c index aad89ba16b9b..cb48a9477514 100644 --- a/fs/bcachefs/btree_io.c +++ b/fs/bcachefs/btree_io.c @@ -1749,10 +1749,8 @@ static int __bch2_btree_root_read(struct btree_trans *trans, enum btree_id id, bch2_btree_node_read(trans, b, true); if (btree_node_read_error(b)) { - bch2_btree_node_hash_remove(&c->btree_cache, b); - mutex_lock(&c->btree_cache.lock); - list_move(&b->list, &c->btree_cache.freeable); + bch2_btree_node_hash_remove(&c->btree_cache, b); mutex_unlock(&c->btree_cache.lock); ret = -BCH_ERR_btree_node_read_error; diff --git a/fs/bcachefs/btree_types.h b/fs/bcachefs/btree_types.h index 806d27b7f41b..ee3df2a486cc 100644 --- a/fs/bcachefs/btree_types.h +++ b/fs/bcachefs/btree_types.h @@ -179,8 +179,8 @@ struct btree_cache { struct list_head freed_pcpu; struct list_head freed_nonpcpu; - /* Number of elements in live + freeable lists */ - size_t nr_used; + size_t nr_live; + size_t nr_freeable; size_t nr_reserve; size_t nr_by_btree[BTREE_ID_NR]; atomic_long_t nr_dirty; diff --git a/fs/bcachefs/btree_update_interior.c b/fs/bcachefs/btree_update_interior.c index 24e445574ab8..18494a662e0a 100644 --- a/fs/bcachefs/btree_update_interior.c +++ b/fs/bcachefs/btree_update_interior.c @@ -251,8 +251,13 @@ static void bch2_btree_node_free_inmem(struct btree_trans *trans, unsigned i, level = b->c.level; bch2_btree_node_lock_write_nofail(trans, path, &b->c); + + mutex_lock(&c->btree_cache.lock); bch2_btree_node_hash_remove(&c->btree_cache, b); + mutex_unlock(&c->btree_cache.lock); + __btree_node_free(trans, b); + six_unlock_write(&b->c.lock); mark_btree_node_locked_noreset(path, level, BTREE_NODE_INTENT_LOCKED); @@ -284,7 +289,6 @@ static void bch2_btree_node_free_never_used(struct btree_update *as, clear_btree_node_need_write(b); mutex_lock(&c->btree_cache.lock); - list_del_init(&b->list); bch2_btree_node_hash_remove(&c->btree_cache, b); mutex_unlock(&c->btree_cache.lock); diff --git a/fs/bcachefs/journal_reclaim.c b/fs/bcachefs/journal_reclaim.c index 9794b6d214cd..f8e045982753 100644 --- a/fs/bcachefs/journal_reclaim.c +++ b/fs/bcachefs/journal_reclaim.c @@ -681,7 +681,7 @@ static int __bch2_journal_reclaim(struct journal *j, bool direct, bool kicked) if (j->watermark != BCH_WATERMARK_stripe) min_nr = 1; - if (atomic_long_read(&c->btree_cache.nr_dirty) * 2 > c->btree_cache.nr_used) + if (atomic_long_read(&c->btree_cache.nr_dirty) * 2 > c->btree_cache.nr_live) min_nr = 1; min_key_cache = min(bch2_nr_btree_keys_need_flush(c), (size_t) 128); @@ -690,7 +690,7 @@ static int __bch2_journal_reclaim(struct journal *j, bool direct, bool kicked) direct, kicked, min_nr, min_key_cache, atomic_long_read(&c->btree_cache.nr_dirty), - c->btree_cache.nr_used, + c->btree_cache.nr_live, atomic_long_read(&c->btree_key_cache.nr_dirty), atomic_long_read(&c->btree_key_cache.nr_keys)); From 7a51608d0125469664e2daf8e060d6d783924c98 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 4 Sep 2024 20:49:37 -0400 Subject: [PATCH 288/302] bcachefs: Rework btree node pinning In backpointers fsck, we do a seqential scan of one btree, and check references to another: extents <-> backpointers Checking references generates random lookups, so we want to pin that btree in memory (or only a range, if it doesn't fit in ram). Previously, this was done with a simple check in the shrinker - "if btree node is in range being pinned, don't free it" - but this generated OOMs, as our shrinker wasn't well behaved if there was less memory available than expected. Instead, we now have two different shrinkers and lru lists; the second shrinker being for pinned nodes, with seeks set much higher than normal - so they can still be freed if necessary, but we'll prefer not to. Signed-off-by: Kent Overstreet --- fs/bcachefs/backpointers.c | 13 ++- fs/bcachefs/btree_cache.c | 164 +++++++++++++++++++--------- fs/bcachefs/btree_cache.h | 3 + fs/bcachefs/btree_types.h | 21 ++-- fs/bcachefs/btree_update_interior.c | 2 +- fs/bcachefs/journal_reclaim.c | 7 +- fs/bcachefs/sysfs.c | 15 ++- 7 files changed, 150 insertions(+), 75 deletions(-) diff --git a/fs/bcachefs/backpointers.c b/fs/bcachefs/backpointers.c index 6c395445f60c..b88d1801652c 100644 --- a/fs/bcachefs/backpointers.c +++ b/fs/bcachefs/backpointers.c @@ -752,10 +752,12 @@ static int bch2_get_btree_in_memory_pos(struct btree_trans *trans, s64 mem_may_pin = mem_may_pin_bytes(c); int ret = 0; + bch2_btree_cache_unpin(c); + btree_interior_mask |= btree_leaf_mask; - c->btree_cache.pinned_nodes_leaf_mask = btree_leaf_mask; - c->btree_cache.pinned_nodes_interior_mask = btree_interior_mask; + c->btree_cache.pinned_nodes_mask[0] = btree_leaf_mask; + c->btree_cache.pinned_nodes_mask[1] = btree_interior_mask; c->btree_cache.pinned_nodes_start = start; c->btree_cache.pinned_nodes_end = *end = BBPOS_MAX; @@ -777,6 +779,7 @@ static int bch2_get_btree_in_memory_pos(struct btree_trans *trans, BBPOS(btree, b->key.k.p); break; } + bch2_node_pin(c, b); 0; })); } @@ -936,8 +939,7 @@ int bch2_check_extents_to_backpointers(struct bch_fs *c) bch2_trans_put(trans); bch2_bkey_buf_exit(&s.last_flushed, c); - c->btree_cache.pinned_nodes_leaf_mask = 0; - c->btree_cache.pinned_nodes_interior_mask = 0; + bch2_btree_cache_unpin(c); bch_err_fn(c, ret); return ret; @@ -1053,8 +1055,7 @@ int bch2_check_backpointers_to_extents(struct bch_fs *c) } bch2_trans_put(trans); - c->btree_cache.pinned_nodes_leaf_mask = 0; - c->btree_cache.pinned_nodes_interior_mask = 0; + bch2_btree_cache_unpin(c); bch_err_fn(c, ret); return ret; diff --git a/fs/bcachefs/btree_cache.c b/fs/bcachefs/btree_cache.c index e66853b98857..6e4afb2b5441 100644 --- a/fs/bcachefs/btree_cache.c +++ b/fs/bcachefs/btree_cache.c @@ -47,9 +47,14 @@ void bch2_recalc_btree_reserve(struct bch_fs *c) c->btree_cache.nr_reserve = reserve; } -static inline size_t btree_cache_can_free(struct btree_cache *bc) +static inline size_t btree_cache_can_free(struct btree_cache_list *list) { - return max_t(int, 0, bc->nr_live + bc->nr_freeable - bc->nr_reserve); + struct btree_cache *bc = container_of(list, struct btree_cache, live[list->idx]); + + size_t can_free = list->nr; + if (!list->idx) + can_free = max_t(ssize_t, 0, can_free - bc->nr_reserve); + return can_free; } static void btree_node_to_freedlist(struct btree_cache *bc, struct btree *b) @@ -184,6 +189,51 @@ void bch2_btree_node_to_freelist(struct bch_fs *c, struct btree *b) six_unlock_intent(&b->c.lock); } +static inline bool __btree_node_pinned(struct btree_cache *bc, struct btree *b) +{ + struct bbpos pos = BBPOS(b->c.btree_id, b->key.k.p); + + u64 mask = bc->pinned_nodes_mask[!!b->c.level]; + + return ((mask & BIT_ULL(b->c.btree_id)) && + bbpos_cmp(bc->pinned_nodes_start, pos) < 0 && + bbpos_cmp(bc->pinned_nodes_end, pos) >= 0); +} + +void bch2_node_pin(struct bch_fs *c, struct btree *b) +{ + struct btree_cache *bc = &c->btree_cache; + + mutex_lock(&bc->lock); + BUG_ON(!__btree_node_pinned(bc, b)); + if (b != btree_node_root(c, b) && !btree_node_pinned(b)) { + set_btree_node_pinned(b); + list_move(&b->list, &bc->live[1].list); + bc->live[0].nr--; + bc->live[1].nr++; + } + mutex_unlock(&bc->lock); +} + +void bch2_btree_cache_unpin(struct bch_fs *c) +{ + struct btree_cache *bc = &c->btree_cache; + struct btree *b, *n; + + mutex_lock(&bc->lock); + c->btree_cache.pinned_nodes_mask[0] = 0; + c->btree_cache.pinned_nodes_mask[1] = 0; + + list_for_each_entry_safe(b, n, &bc->live[1].list, list) { + clear_btree_node_pinned(b); + list_move(&b->list, &bc->live[0].list); + bc->live[0].nr++; + bc->live[1].nr--; + } + + mutex_unlock(&bc->lock); +} + /* Btree in memory cache - hash table */ void bch2_btree_node_hash_remove(struct btree_cache *bc, struct btree *b) @@ -199,7 +249,7 @@ void bch2_btree_node_hash_remove(struct btree_cache *bc, struct btree *b) if (b->c.btree_id < BTREE_ID_NR) --bc->nr_by_btree[b->c.btree_id]; - bc->nr_live--; + bc->live[btree_node_pinned(b)].nr--; bc->nr_freeable++; list_move(&b->list, &bc->freeable); } @@ -216,9 +266,14 @@ int __bch2_btree_node_hash_insert(struct btree_cache *bc, struct btree *b) if (b->c.btree_id < BTREE_ID_NR) bc->nr_by_btree[b->c.btree_id]++; - bc->nr_live++; + + bool p = __btree_node_pinned(bc, b); + mod_bit(BTREE_NODE_pinned, &b->flags, p); + + list_move_tail(&b->list, &bc->live[p].list); + bc->live[p].nr++; + bc->nr_freeable--; - list_move_tail(&b->list, &bc->live); return 0; } @@ -283,20 +338,6 @@ static int __btree_node_reclaim(struct bch_fs *c, struct btree *b, bool flush, b int ret = 0; lockdep_assert_held(&bc->lock); - - struct bbpos pos = BBPOS(b->c.btree_id, b->key.k.p); - - u64 mask = b->c.level - ? bc->pinned_nodes_interior_mask - : bc->pinned_nodes_leaf_mask; - - if ((mask & BIT_ULL(b->c.btree_id)) && - bbpos_cmp(bc->pinned_nodes_start, pos) < 0 && - bbpos_cmp(bc->pinned_nodes_end, pos) >= 0) { - BTREE_CACHE_NOT_FREED_INCREMENT(pinned); - return -BCH_ERR_ENOMEM_btree_node_reclaim; - } - wait_on_io: if (b->flags & ((1U << BTREE_NODE_dirty)| (1U << BTREE_NODE_read_in_flight)| @@ -401,8 +442,9 @@ static int btree_node_write_and_reclaim(struct bch_fs *c, struct btree *b) static unsigned long bch2_btree_cache_scan(struct shrinker *shrink, struct shrink_control *sc) { - struct bch_fs *c = shrink->private_data; - struct btree_cache *bc = &c->btree_cache; + struct btree_cache_list *list = shrink->private_data; + struct btree_cache *bc = container_of(list, struct btree_cache, live[list->idx]); + struct bch_fs *c = container_of(bc, struct bch_fs, btree_cache); struct btree *b, *t; unsigned long nr = sc->nr_to_scan; unsigned long can_free = 0; @@ -410,8 +452,7 @@ static unsigned long bch2_btree_cache_scan(struct shrinker *shrink, unsigned long touched = 0; unsigned i, flags; unsigned long ret = SHRINK_STOP; - bool trigger_writes = atomic_long_read(&bc->nr_dirty) + nr >= - (bc->nr_live + bc->nr_freeable) * 3 / 4; + bool trigger_writes = atomic_long_read(&bc->nr_dirty) + nr >= list->nr * 3 / 4; if (bch2_btree_shrinker_disabled) return SHRINK_STOP; @@ -426,7 +467,7 @@ static unsigned long bch2_btree_cache_scan(struct shrinker *shrink, * succeed, so that inserting keys into the btree can always succeed and * IO can always make forward progress: */ - can_free = btree_cache_can_free(bc); + can_free = btree_cache_can_free(list); nr = min_t(unsigned long, nr, can_free); i = 0; @@ -452,7 +493,7 @@ static unsigned long bch2_btree_cache_scan(struct shrinker *shrink, } } restart: - list_for_each_entry_safe(b, t, &bc->live, list) { + list_for_each_entry_safe(b, t, &list->list, list) { touched++; if (btree_node_accessed(b)) { @@ -476,7 +517,7 @@ restart: !btree_node_will_make_reachable(b) && !btree_node_write_blocked(b) && six_trylock_read(&b->c.lock)) { - list_move(&bc->live, &b->list); + list_move(&list->list, &b->list); mutex_unlock(&bc->lock); __bch2_btree_node_write(c, b, BTREE_WRITE_cache_reclaim); six_unlock_read(&b->c.lock); @@ -490,8 +531,8 @@ restart: break; } out_rotate: - if (&t->list != &bc->live) - list_move_tail(&bc->live, &t->list); + if (&t->list != &list->list) + list_move_tail(&list->list, &t->list); out: mutex_unlock(&bc->lock); out_nounlock: @@ -504,40 +545,42 @@ out_nounlock: static unsigned long bch2_btree_cache_count(struct shrinker *shrink, struct shrink_control *sc) { - struct bch_fs *c = shrink->private_data; - struct btree_cache *bc = &c->btree_cache; + struct btree_cache_list *list = shrink->private_data; if (bch2_btree_shrinker_disabled) return 0; - return btree_cache_can_free(bc); + return btree_cache_can_free(list); } void bch2_fs_btree_cache_exit(struct bch_fs *c) { struct btree_cache *bc = &c->btree_cache; struct btree *b, *t; - unsigned i, flags; + unsigned long flags; - shrinker_free(bc->shrink); + shrinker_free(bc->live[1].shrink); + shrinker_free(bc->live[0].shrink); /* vfree() can allocate memory: */ flags = memalloc_nofs_save(); mutex_lock(&bc->lock); if (c->verify_data) - list_move(&c->verify_data->list, &bc->live); + list_move(&c->verify_data->list, &bc->live[0].list); kvfree(c->verify_ondisk); - for (i = 0; i < btree_id_nr_alive(c); i++) { + for (unsigned i = 0; i < btree_id_nr_alive(c); i++) { struct btree_root *r = bch2_btree_id_root(c, i); if (r->b) - list_add(&r->b->list, &bc->live); + list_add(&r->b->list, &bc->live[0].list); } - list_for_each_entry_safe(b, t, &bc->live, list) + list_for_each_entry_safe(b, t, &bc->live[1].list, list) + bch2_btree_node_hash_remove(bc, b); + list_for_each_entry_safe(b, t, &bc->live[0].list, list) bch2_btree_node_hash_remove(bc, b); list_for_each_entry_safe(b, t, &bc->freeable, list) { @@ -563,7 +606,8 @@ void bch2_fs_btree_cache_exit(struct bch_fs *c) for (unsigned i = 0; i < ARRAY_SIZE(bc->nr_by_btree); i++) BUG_ON(bc->nr_by_btree[i]); - BUG_ON(bc->nr_live); + BUG_ON(bc->live[0].nr); + BUG_ON(bc->live[1].nr); BUG_ON(bc->nr_freeable); if (bc->table_init_done) @@ -589,18 +633,28 @@ int bch2_fs_btree_cache_init(struct bch_fs *c) if (!__bch2_btree_node_mem_alloc(c)) goto err; - list_splice_init(&bc->live, &bc->freeable); + list_splice_init(&bc->live[0].list, &bc->freeable); mutex_init(&c->verify_lock); shrink = shrinker_alloc(0, "%s-btree_cache", c->name); if (!shrink) goto err; - bc->shrink = shrink; + bc->live[0].shrink = shrink; shrink->count_objects = bch2_btree_cache_count; shrink->scan_objects = bch2_btree_cache_scan; - shrink->seeks = 4; - shrink->private_data = c; + shrink->seeks = 2; + shrink->private_data = &bc->live[0]; + shrinker_register(shrink); + + shrink = shrinker_alloc(0, "%s-btree_cache-pinned", c->name); + if (!shrink) + goto err; + bc->live[1].shrink = shrink; + shrink->count_objects = bch2_btree_cache_count; + shrink->scan_objects = bch2_btree_cache_scan; + shrink->seeks = 8; + shrink->private_data = &bc->live[1]; shrinker_register(shrink); return 0; @@ -611,7 +665,10 @@ err: void bch2_fs_btree_cache_init_early(struct btree_cache *bc) { mutex_init(&bc->lock); - INIT_LIST_HEAD(&bc->live); + for (unsigned i = 0; i < ARRAY_SIZE(bc->live); i++) { + bc->live[i].idx = i; + INIT_LIST_HEAD(&bc->live[i].list); + } INIT_LIST_HEAD(&bc->freeable); INIT_LIST_HEAD(&bc->freed_pcpu); INIT_LIST_HEAD(&bc->freed_nonpcpu); @@ -673,14 +730,16 @@ static struct btree *btree_node_cannibalize(struct bch_fs *c) struct btree_cache *bc = &c->btree_cache; struct btree *b; - list_for_each_entry_reverse(b, &bc->live, list) - if (!btree_node_reclaim(c, b, false)) - return b; + for (unsigned i = 0; i < ARRAY_SIZE(bc->live); i++) + list_for_each_entry_reverse(b, &bc->live[i].list, list) + if (!btree_node_reclaim(c, b, false)) + return b; while (1) { - list_for_each_entry_reverse(b, &bc->live, list) - if (!btree_node_write_and_reclaim(c, b)) - return b; + for (unsigned i = 0; i < ARRAY_SIZE(bc->live); i++) + list_for_each_entry_reverse(b, &bc->live[i].list, list) + if (!btree_node_write_and_reclaim(c, b)) + return b; /* * Rare case: all nodes were intent-locked. @@ -1387,9 +1446,10 @@ void bch2_btree_cache_to_text(struct printbuf *out, const struct btree_cache *bc if (!out->nr_tabstops) printbuf_tabstop_push(out, 32); - prt_btree_cache_line(out, c, "nr_live:", bc->nr_live); - prt_btree_cache_line(out, c, "nr_freeable:", bc->nr_freeable); - prt_btree_cache_line(out, c, "nr dirty:", atomic_long_read(&bc->nr_dirty)); + prt_btree_cache_line(out, c, "live:", bc->live[0].nr); + prt_btree_cache_line(out, c, "pinned:", bc->live[1].nr); + prt_btree_cache_line(out, c, "freeable:", bc->nr_freeable); + prt_btree_cache_line(out, c, "dirty:", atomic_long_read(&bc->nr_dirty)); prt_printf(out, "cannibalize lock:\t%p\n", bc->alloc_lock); prt_newline(out); diff --git a/fs/bcachefs/btree_cache.h b/fs/bcachefs/btree_cache.h index f82064007127..367acd217c6a 100644 --- a/fs/bcachefs/btree_cache.h +++ b/fs/bcachefs/btree_cache.h @@ -19,6 +19,9 @@ int __bch2_btree_node_hash_insert(struct btree_cache *, struct btree *); int bch2_btree_node_hash_insert(struct btree_cache *, struct btree *, unsigned, enum btree_id); +void bch2_node_pin(struct bch_fs *, struct btree *); +void bch2_btree_cache_unpin(struct bch_fs *); + void bch2_btree_node_update_key_early(struct btree_trans *, enum btree_id, unsigned, struct bkey_s_c, struct bkey_i *); diff --git a/fs/bcachefs/btree_types.h b/fs/bcachefs/btree_types.h index ee3df2a486cc..4568a41fefaf 100644 --- a/fs/bcachefs/btree_types.h +++ b/fs/bcachefs/btree_types.h @@ -147,8 +147,7 @@ struct btree { x(noevict) \ x(write_blocked) \ x(will_make_reachable) \ - x(access_bit) \ - x(pinned) \ + x(access_bit) enum bch_btree_cache_not_freed_reasons { #define x(n) BCH_BTREE_CACHE_NOT_FREED_##n, @@ -157,6 +156,13 @@ enum bch_btree_cache_not_freed_reasons { BCH_BTREE_CACHE_NOT_FREED_REASONS_NR, }; +struct btree_cache_list { + unsigned idx; + struct shrinker *shrink; + struct list_head list; + size_t nr; +}; + struct btree_cache { struct rhashtable table; bool table_init_done; @@ -174,12 +180,11 @@ struct btree_cache { * should never grow past ~2-3 nodes in practice. */ struct mutex lock; - struct list_head live; struct list_head freeable; struct list_head freed_pcpu; struct list_head freed_nonpcpu; + struct btree_cache_list live[2]; - size_t nr_live; size_t nr_freeable; size_t nr_reserve; size_t nr_by_btree[BTREE_ID_NR]; @@ -188,7 +193,6 @@ struct btree_cache { /* shrinker stats */ size_t nr_freed; u64 not_freed[BCH_BTREE_CACHE_NOT_FREED_REASONS_NR]; - struct shrinker *shrink; /* * If we need to allocate memory for a new btree node and that @@ -201,8 +205,8 @@ struct btree_cache { struct bbpos pinned_nodes_start; struct bbpos pinned_nodes_end; - u64 pinned_nodes_leaf_mask; - u64 pinned_nodes_interior_mask; + /* btree id mask: 0 for leaves, 1 for interior */ + u64 pinned_nodes_mask[2]; }; struct btree_node_iter { @@ -594,7 +598,8 @@ enum btree_write_type { x(dying) \ x(fake) \ x(need_rewrite) \ - x(never_write) + x(never_write) \ + x(pinned) enum btree_flags { /* First bits for btree node write type */ diff --git a/fs/bcachefs/btree_update_interior.c b/fs/bcachefs/btree_update_interior.c index 18494a662e0a..190bc1e81756 100644 --- a/fs/bcachefs/btree_update_interior.c +++ b/fs/bcachefs/btree_update_interior.c @@ -1904,7 +1904,7 @@ static void __btree_increase_depth(struct btree_update *as, struct btree_trans * six_unlock_intent(&n->c.lock); mutex_lock(&c->btree_cache.lock); - list_add_tail(&b->list, &c->btree_cache.live); + list_add_tail(&b->list, &c->btree_cache.live[btree_node_pinned(b)].list); mutex_unlock(&c->btree_cache.lock); bch2_trans_verify_locks(trans); diff --git a/fs/bcachefs/journal_reclaim.c b/fs/bcachefs/journal_reclaim.c index f8e045982753..ace291f175dd 100644 --- a/fs/bcachefs/journal_reclaim.c +++ b/fs/bcachefs/journal_reclaim.c @@ -641,6 +641,7 @@ static u64 journal_seq_to_flush(struct journal *j) static int __bch2_journal_reclaim(struct journal *j, bool direct, bool kicked) { struct bch_fs *c = container_of(j, struct bch_fs, journal); + struct btree_cache *bc = &c->btree_cache; bool kthread = (current->flags & PF_KTHREAD) != 0; u64 seq_to_flush; size_t min_nr, min_key_cache, nr_flushed; @@ -681,7 +682,8 @@ static int __bch2_journal_reclaim(struct journal *j, bool direct, bool kicked) if (j->watermark != BCH_WATERMARK_stripe) min_nr = 1; - if (atomic_long_read(&c->btree_cache.nr_dirty) * 2 > c->btree_cache.nr_live) + size_t btree_cache_live = bc->live[0].nr + bc->live[1].nr; + if (atomic_long_read(&bc->nr_dirty) * 2 > btree_cache_live) min_nr = 1; min_key_cache = min(bch2_nr_btree_keys_need_flush(c), (size_t) 128); @@ -689,8 +691,7 @@ static int __bch2_journal_reclaim(struct journal *j, bool direct, bool kicked) trace_and_count(c, journal_reclaim_start, c, direct, kicked, min_nr, min_key_cache, - atomic_long_read(&c->btree_cache.nr_dirty), - c->btree_cache.nr_live, + atomic_long_read(&bc->nr_dirty), btree_cache_live, atomic_long_read(&c->btree_key_cache.nr_dirty), atomic_long_read(&c->btree_key_cache.nr_keys)); diff --git a/fs/bcachefs/sysfs.c b/fs/bcachefs/sysfs.c index 6791540d6a4a..03e59f86f360 100644 --- a/fs/bcachefs/sysfs.c +++ b/fs/bcachefs/sysfs.c @@ -244,14 +244,18 @@ static struct attribute sysfs_state_rw = { static size_t bch2_btree_cache_size(struct bch_fs *c) { + struct btree_cache *bc = &c->btree_cache; size_t ret = 0; struct btree *b; - mutex_lock(&c->btree_cache.lock); - list_for_each_entry(b, &c->btree_cache.live, list) + mutex_lock(&bc->lock); + list_for_each_entry(b, &bc->live[0].list, list) ret += btree_buf_bytes(b); - - mutex_unlock(&c->btree_cache.lock); + list_for_each_entry(b, &bc->live[1].list, list) + ret += btree_buf_bytes(b); + list_for_each_entry(b, &bc->freeable, list) + ret += btree_buf_bytes(b); + mutex_unlock(&bc->lock); return ret; } @@ -444,11 +448,12 @@ STORE(bch2_fs) return -EROFS; if (attr == &sysfs_trigger_btree_cache_shrink) { + struct btree_cache *bc = &c->btree_cache; struct shrink_control sc; sc.gfp_mask = GFP_KERNEL; sc.nr_to_scan = strtoul_or_return(buf); - c->btree_cache.shrink->scan_objects(c->btree_cache.shrink, &sc); + bc->live[0].shrink->scan_objects(bc->live[0].shrink, &sc); } if (attr == &sysfs_trigger_btree_key_cache_shrink) { From 54a12984a9a29d6b7a363e8860506f0c77f334d7 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 4 Sep 2024 17:51:47 -0400 Subject: [PATCH 289/302] bcachefs: EIO errcode cleanup We want to be using private errcodes whenever possible, for better error messages. Signed-off-by: Kent Overstreet --- fs/bcachefs/buckets.c | 14 +++++++------- fs/bcachefs/ec.c | 24 ++++++++++++------------ fs/bcachefs/errcode.h | 9 +++++++++ fs/bcachefs/extents.c | 11 ++++------- fs/bcachefs/io_read.c | 2 +- 5 files changed, 33 insertions(+), 27 deletions(-) diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c index 2d1d4dae631b..0732ac8f682d 100644 --- a/fs/bcachefs/buckets.c +++ b/fs/bcachefs/buckets.c @@ -486,7 +486,7 @@ out: return ret; err: bch2_dump_trans_updates(trans); - ret = -EIO; + ret = -BCH_ERR_bucket_ref_update; goto out; } @@ -573,7 +573,7 @@ static int bch2_trigger_pointer(struct btree_trans *trans, struct bch_dev *ca = bch2_dev_tryget(c, p.ptr.dev); if (unlikely(!ca)) { if (insert && p.ptr.dev != BCH_SB_MEMBER_INVALID) - ret = -EIO; + ret = -BCH_ERR_trigger_pointer; goto err; } @@ -602,7 +602,7 @@ static int bch2_trigger_pointer(struct btree_trans *trans, if (bch2_fs_inconsistent_on(!g, c, "reference to invalid bucket on device %u\n %s", p.ptr.dev, (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) { - ret = -EIO; + ret = -BCH_ERR_trigger_pointer; goto err_unlock; } @@ -647,7 +647,7 @@ static int bch2_trigger_stripe_ptr(struct btree_trans *trans, bch2_trans_inconsistent(trans, "stripe pointer doesn't match stripe %llu", (u64) p.ec.idx); - ret = -EIO; + ret = -BCH_ERR_trigger_stripe_pointer; goto err; } @@ -686,7 +686,7 @@ err: (u64) p.ec.idx, buf.buf); printbuf_exit(&buf); bch2_inconsistent_error(c); - return -EIO; + return -BCH_ERR_trigger_stripe_pointer; } m->block_sectors[p.ec.block] += sectors; @@ -966,7 +966,7 @@ static int __bch2_trans_mark_metadata_bucket(struct btree_trans *trans, bch2_data_type_str(a->v.data_type), bch2_data_type_str(type), bch2_data_type_str(type)); - ret = -EIO; + ret = -BCH_ERR_metadata_bucket_inconsistency; goto err; } @@ -1022,7 +1022,7 @@ err: bucket_unlock(g); err_unlock: percpu_up_read(&c->mark_lock); - return -EIO; + return -BCH_ERR_metadata_bucket_inconsistency; } int bch2_trans_mark_metadata_bucket(struct btree_trans *trans, diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index fa1be6a29d33..0812d2c76384 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -193,7 +193,7 @@ static int __mark_stripe_bucket(struct btree_trans *trans, a->dirty_sectors, a->stripe, s.k->p.offset, (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf))) { - ret = -EIO; + ret = -BCH_ERR_mark_stripe; goto err; } @@ -204,7 +204,7 @@ static int __mark_stripe_bucket(struct btree_trans *trans, a->dirty_sectors, a->cached_sectors, (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf))) { - ret = -EIO; + ret = -BCH_ERR_mark_stripe; goto err; } } else { @@ -214,7 +214,7 @@ static int __mark_stripe_bucket(struct btree_trans *trans, bucket.inode, bucket.offset, a->gen, a->stripe, (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf))) { - ret = -EIO; + ret = -BCH_ERR_mark_stripe; goto err; } @@ -224,7 +224,7 @@ static int __mark_stripe_bucket(struct btree_trans *trans, bch2_data_type_str(a->data_type), bch2_data_type_str(data_type), (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf))) { - ret = -EIO; + ret = -BCH_ERR_mark_stripe; goto err; } @@ -236,7 +236,7 @@ static int __mark_stripe_bucket(struct btree_trans *trans, a->dirty_sectors, a->cached_sectors, (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf))) { - ret = -EIO; + ret = -BCH_ERR_mark_stripe; goto err; } } @@ -274,8 +274,8 @@ static int mark_stripe_bucket(struct btree_trans *trans, struct bch_dev *ca = bch2_dev_tryget(c, ptr->dev); if (unlikely(!ca)) { - if (!(flags & BTREE_TRIGGER_overwrite)) - ret = -EIO; + if (ptr->dev != BCH_SB_MEMBER_INVALID && !(flags & BTREE_TRIGGER_overwrite)) + ret = -BCH_ERR_mark_stripe; goto err; } @@ -294,7 +294,7 @@ static int mark_stripe_bucket(struct btree_trans *trans, if (bch2_fs_inconsistent_on(!g, c, "reference to invalid bucket on device %u\n %s", ptr->dev, (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf))) { - ret = -EIO; + ret = -BCH_ERR_mark_stripe; goto err_unlock; } @@ -839,7 +839,7 @@ int bch2_ec_read_extent(struct btree_trans *trans, struct bch_read_bio *rbio) bch_err_ratelimited(c, "error doing reconstruct read: error %i looking up stripe", ret); kfree(buf); - return -EIO; + return -BCH_ERR_stripe_reconstruct; } v = &bkey_i_to_stripe(&buf->key)->v; @@ -847,7 +847,7 @@ int bch2_ec_read_extent(struct btree_trans *trans, struct bch_read_bio *rbio) if (!bch2_ptr_matches_stripe(v, rbio->pick)) { bch_err_ratelimited(c, "error doing reconstruct read: pointer doesn't match stripe"); - ret = -EIO; + ret = -BCH_ERR_stripe_reconstruct; goto err; } @@ -855,7 +855,7 @@ int bch2_ec_read_extent(struct btree_trans *trans, struct bch_read_bio *rbio) if (offset + bio_sectors(&rbio->bio) > le16_to_cpu(v->sectors)) { bch_err_ratelimited(c, "error doing reconstruct read: read is bigger than stripe"); - ret = -EIO; + ret = -BCH_ERR_stripe_reconstruct; goto err; } @@ -871,7 +871,7 @@ int bch2_ec_read_extent(struct btree_trans *trans, struct bch_read_bio *rbio) if (ec_nr_failed(buf) > v->nr_redundant) { bch_err_ratelimited(c, "error doing reconstruct read: unable to read enough blocks"); - ret = -EIO; + ret = -BCH_ERR_stripe_reconstruct; goto err; } diff --git a/fs/bcachefs/errcode.h b/fs/bcachefs/errcode.h index 96be8d600ca0..5241910c1cb0 100644 --- a/fs/bcachefs/errcode.h +++ b/fs/bcachefs/errcode.h @@ -244,6 +244,15 @@ x(EIO, btree_node_read_error) \ x(EIO, btree_node_read_validate_error) \ x(EIO, btree_need_topology_repair) \ + x(EIO, bucket_ref_update) \ + x(EIO, trigger_pointer) \ + x(EIO, trigger_stripe_pointer) \ + x(EIO, metadata_bucket_inconsistency) \ + x(EIO, mark_stripe) \ + x(EIO, stripe_reconstruct) \ + x(EIO, key_type_error) \ + x(EIO, no_device_to_read_from) \ + x(EIO, missing_indirect_extent) \ x(BCH_ERR_btree_node_read_err, btree_node_read_err_fixable) \ x(BCH_ERR_btree_node_read_err, btree_node_read_err_want_retry) \ x(BCH_ERR_btree_node_read_err, btree_node_read_err_must_retry) \ diff --git a/fs/bcachefs/extents.c b/fs/bcachefs/extents.c index ad63445b85e5..d370d679e015 100644 --- a/fs/bcachefs/extents.c +++ b/fs/bcachefs/extents.c @@ -115,7 +115,7 @@ int bch2_bkey_pick_read_device(struct bch_fs *c, struct bkey_s_c k, int ret = 0; if (k.k->type == KEY_TYPE_error) - return -EIO; + return -BCH_ERR_key_type_error; rcu_read_lock(); bkey_for_each_ptr_decode(k.k, ptrs, p, entry) { @@ -133,7 +133,7 @@ int bch2_bkey_pick_read_device(struct bch_fs *c, struct bkey_s_c k, * read: */ if (!ret && !p.ptr.cached) - ret = -EIO; + ret = -BCH_ERR_no_device_to_read_from; struct bch_dev *ca = bch2_dev_rcu(c, p.ptr.dev); @@ -146,16 +146,13 @@ int bch2_bkey_pick_read_device(struct bch_fs *c, struct bkey_s_c k, ? f->idx : f->idx + 1; - if (!p.idx && !ca) + if (!p.idx && (!ca || !bch2_dev_is_readable(ca))) p.idx++; if (!p.idx && p.has_ec && bch2_force_reconstruct_read) p.idx++; - if (!p.idx && !bch2_dev_is_readable(ca)) - p.idx++; - - if (p.idx >= (unsigned) p.has_ec + 1) + if (p.idx > (unsigned) p.has_ec) continue; if (ret > 0 && !ptr_better(c, p, *pick)) diff --git a/fs/bcachefs/io_read.c b/fs/bcachefs/io_read.c index 10242671b269..b3fba16b157b 100644 --- a/fs/bcachefs/io_read.c +++ b/fs/bcachefs/io_read.c @@ -777,7 +777,7 @@ int __bch2_read_indirect_extent(struct btree_trans *trans, orig_k->k->k.size, reflink_offset); bch2_inconsistent_error(trans->c); - ret = -EIO; + ret = -BCH_ERR_missing_indirect_extent; goto err; } From 1b11c4d3654877f4deb6f9bec12f6ec487d62030 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 1 Sep 2024 14:51:45 -0400 Subject: [PATCH 290/302] bcachefs: stripe_to_mem() factor out a common helper Signed-off-by: Kent Overstreet --- fs/bcachefs/ec.c | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index 0812d2c76384..9efffc43406d 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -352,6 +352,18 @@ static int mark_stripe_buckets(struct btree_trans *trans, return 0; } +static inline void stripe_to_mem(struct stripe *m, const struct bch_stripe *s) +{ + m->sectors = le16_to_cpu(s->sectors); + m->algorithm = s->algorithm; + m->nr_blocks = s->nr_blocks; + m->nr_redundant = s->nr_redundant; + m->blocks_nonempty = 0; + + for (unsigned i = 0; i < s->nr_blocks; i++) + m->blocks_nonempty += !!stripe_blockcount_get(s, i); +} + int bch2_trigger_stripe(struct btree_trans *trans, enum btree_id btree, unsigned level, struct bkey_s_c old, struct bkey_s _new, @@ -468,14 +480,7 @@ int bch2_trigger_stripe(struct btree_trans *trans, memset(m, 0, sizeof(*m)); } else { - m->sectors = le16_to_cpu(new_s->sectors); - m->algorithm = new_s->algorithm; - m->nr_blocks = new_s->nr_blocks; - m->nr_redundant = new_s->nr_redundant; - m->blocks_nonempty = 0; - - for (unsigned i = 0; i < new_s->nr_blocks; i++) - m->blocks_nonempty += !!stripe_blockcount_get(new_s, i); + stripe_to_mem(m, new_s); if (!old_s) bch2_stripes_heap_insert(c, m, idx); @@ -2198,17 +2203,9 @@ int bch2_stripes_read(struct bch_fs *c) if (ret) break; - const struct bch_stripe *s = bkey_s_c_to_stripe(k).v; - struct stripe *m = genradix_ptr(&c->stripes, k.k->p.offset); - m->sectors = le16_to_cpu(s->sectors); - m->algorithm = s->algorithm; - m->nr_blocks = s->nr_blocks; - m->nr_redundant = s->nr_redundant; - m->blocks_nonempty = 0; - for (unsigned i = 0; i < s->nr_blocks; i++) - m->blocks_nonempty += !!stripe_blockcount_get(s, i); + stripe_to_mem(m, bkey_s_c_to_stripe(k).v); bch2_stripes_heap_insert(c, m, k.k->p.offset); 0; From fa85c473977c0afb5104a630f019f593fe33a8b0 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 1 Sep 2024 14:54:42 -0400 Subject: [PATCH 291/302] bcachefs: bch_stripe.disk_label When reshaping existing stripes, we should keep them on the same target that they were allocated on; to do this, we need to add a field to the btree stripe type. This is a tad awkward, because we only have 8 bits left, and targets are 16 bits - but we only need to store a label, not a full target. Signed-off-by: Kent Overstreet --- fs/bcachefs/ec.c | 47 +++++++++++++++++++++++++++++------------ fs/bcachefs/ec.h | 2 +- fs/bcachefs/ec_format.h | 9 +++++++- fs/bcachefs/ec_types.h | 1 + 4 files changed, 43 insertions(+), 16 deletions(-) diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index 9efffc43406d..50e5fdc30874 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -147,12 +147,18 @@ void bch2_stripe_to_text(struct printbuf *out, struct bch_fs *c, bch2_prt_csum_type(out, s.csum_type); prt_printf(out, " gran %u", 1U << s.csum_granularity_bits); + if (s.disk_label) { + prt_str(out, " label"); + bch2_disk_path_to_text(out, c, s.disk_label - 1); + } + for (unsigned i = 0; i < s.nr_blocks; i++) { const struct bch_extent_ptr *ptr = sp->ptrs + i; if ((void *) ptr >= bkey_val_end(k)) break; + prt_char(out, ' '); bch2_extent_ptr_to_text(out, c, ptr); if (s.csum_type < BCH_CSUM_NR && @@ -358,6 +364,7 @@ static inline void stripe_to_mem(struct stripe *m, const struct bch_stripe *s) m->algorithm = s->algorithm; m->nr_blocks = s->nr_blocks; m->nr_redundant = s->nr_redundant; + m->disk_label = s->disk_label; m->blocks_nonempty = 0; for (unsigned i = 0; i < s->nr_blocks; i++) @@ -1647,7 +1654,8 @@ static void ec_stripe_key_init(struct bch_fs *c, struct bkey_i *k, unsigned nr_data, unsigned nr_parity, - unsigned stripe_size) + unsigned stripe_size, + unsigned disk_label) { struct bkey_i_stripe *s = bkey_stripe_init(k); unsigned u64s; @@ -1658,7 +1666,7 @@ static void ec_stripe_key_init(struct bch_fs *c, s->v.nr_redundant = nr_parity; s->v.csum_granularity_bits = ilog2(c->opts.encoded_extent_max >> 9); s->v.csum_type = BCH_CSUM_crc32c; - s->v.pad = 0; + s->v.disk_label = disk_label; while ((u64s = stripe_val_u64s(&s->v)) > BKEY_VAL_U64s_MAX) { BUG_ON(1 << s->v.csum_granularity_bits >= @@ -1691,14 +1699,15 @@ static int ec_new_stripe_alloc(struct bch_fs *c, struct ec_stripe_head *h) s->nr_parity = h->redundancy; ec_stripe_key_init(c, &s->new_stripe.key, - s->nr_data, s->nr_parity, h->blocksize); + s->nr_data, s->nr_parity, + h->blocksize, h->disk_label); h->s = s; return 0; } static struct ec_stripe_head * -ec_new_stripe_head_alloc(struct bch_fs *c, unsigned target, +ec_new_stripe_head_alloc(struct bch_fs *c, unsigned disk_label, unsigned algo, unsigned redundancy, enum bch_watermark watermark) { @@ -1711,13 +1720,13 @@ ec_new_stripe_head_alloc(struct bch_fs *c, unsigned target, mutex_init(&h->lock); BUG_ON(!mutex_trylock(&h->lock)); - h->target = target; + h->disk_label = disk_label; h->algo = algo; h->redundancy = redundancy; h->watermark = watermark; rcu_read_lock(); - h->devs = target_rw_devs(c, BCH_DATA_user, target); + h->devs = target_rw_devs(c, BCH_DATA_user, disk_label ? group_to_target(disk_label - 1) : 0); for_each_member_device_rcu(c, ca, &h->devs) if (!ca->mi.durability) @@ -1756,7 +1765,7 @@ void bch2_ec_stripe_head_put(struct bch_fs *c, struct ec_stripe_head *h) static struct ec_stripe_head * __bch2_ec_stripe_head_get(struct btree_trans *trans, - unsigned target, + unsigned disk_label, unsigned algo, unsigned redundancy, enum bch_watermark watermark) @@ -1778,7 +1787,7 @@ __bch2_ec_stripe_head_get(struct btree_trans *trans, } list_for_each_entry(h, &c->ec_stripe_head_list, list) - if (h->target == target && + if (h->disk_label == disk_label && h->algo == algo && h->redundancy == redundancy && h->watermark == watermark) { @@ -1788,7 +1797,7 @@ __bch2_ec_stripe_head_get(struct btree_trans *trans, goto found; } - h = ec_new_stripe_head_alloc(c, target, algo, redundancy, watermark); + h = ec_new_stripe_head_alloc(c, disk_label, algo, redundancy, watermark); found: if (!IS_ERR_OR_NULL(h) && h->nr_active_devs < h->redundancy + 2) { @@ -1884,7 +1893,6 @@ static int new_stripe_alloc_buckets(struct btree_trans *trans, struct ec_stripe_ return 0; } -/* XXX: doesn't obey target: */ static s64 get_existing_stripe(struct bch_fs *c, struct ec_stripe_head *head) { @@ -1907,7 +1915,8 @@ static s64 get_existing_stripe(struct bch_fs *c, m = genradix_ptr(&c->stripes, stripe_idx); - if (m->algorithm == head->algo && + if (m->disk_label == head->disk_label && + m->algorithm == head->algo && m->nr_redundant == head->redundancy && m->sectors == head->blocksize && m->blocks_nonempty < m->nr_blocks - m->nr_redundant && @@ -2052,9 +2061,19 @@ struct ec_stripe_head *bch2_ec_stripe_head_get(struct btree_trans *trans, struct bch_fs *c = trans->c; struct ec_stripe_head *h; bool waiting = false; + unsigned disk_label = 0; + struct target t = target_decode(target); int ret; - h = __bch2_ec_stripe_head_get(trans, target, algo, redundancy, watermark); + if (t.type == TARGET_GROUP) { + if (t.group > U8_MAX) { + bch_err(c, "cannot create a stripe when disk_label > U8_MAX"); + return NULL; + } + disk_label = t.group + 1; /* 0 == no label */ + } + + h = __bch2_ec_stripe_head_get(trans, disk_label, algo, redundancy, watermark); if (IS_ERR_OR_NULL(h)) return h; @@ -2259,8 +2278,8 @@ void bch2_new_stripes_to_text(struct printbuf *out, struct bch_fs *c) mutex_lock(&c->ec_stripe_head_lock); list_for_each_entry(h, &c->ec_stripe_head_list, list) { - prt_printf(out, "target %u algo %u redundancy %u %s:\n", - h->target, h->algo, h->redundancy, + prt_printf(out, "disk label %u algo %u redundancy %u %s:\n", + h->disk_label, h->algo, h->redundancy, bch2_watermarks[h->watermark]); if (h->s) diff --git a/fs/bcachefs/ec.h b/fs/bcachefs/ec.h index 9baf3411a8f9..b400c7e0ed9c 100644 --- a/fs/bcachefs/ec.h +++ b/fs/bcachefs/ec.h @@ -188,7 +188,7 @@ struct ec_stripe_head { struct list_head list; struct mutex lock; - unsigned target; + unsigned disk_label; unsigned algo; unsigned redundancy; enum bch_watermark watermark; diff --git a/fs/bcachefs/ec_format.h b/fs/bcachefs/ec_format.h index 44ce88ba08d7..64ef52e00078 100644 --- a/fs/bcachefs/ec_format.h +++ b/fs/bcachefs/ec_format.h @@ -11,7 +11,14 @@ struct bch_stripe { __u8 csum_granularity_bits; __u8 csum_type; - __u8 pad; + + /* + * XXX: targets should be 16 bits - fix this if we ever do a stripe_v2 + * + * we can manage with this because this only needs to point to a + * disk label, not a target: + */ + __u8 disk_label; struct bch_extent_ptr ptrs[]; } __packed __aligned(8); diff --git a/fs/bcachefs/ec_types.h b/fs/bcachefs/ec_types.h index 1df03dccfc72..8d1e70e830ac 100644 --- a/fs/bcachefs/ec_types.h +++ b/fs/bcachefs/ec_types.h @@ -16,6 +16,7 @@ struct stripe { u8 nr_blocks; u8 nr_redundant; u8 blocks_nonempty; + u8 disk_label; }; struct gc_stripe { From a4b7a0c0378a3f73004b5c5263ec2b4e5a476fa8 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 1 Sep 2024 16:44:36 -0400 Subject: [PATCH 292/302] bcachefs: ec_stripe_head.nr_created additional debug stat Signed-off-by: Kent Overstreet --- fs/bcachefs/ec.c | 6 ++++-- fs/bcachefs/ec.h | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index 50e5fdc30874..24140c485eee 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -1703,6 +1703,7 @@ static int ec_new_stripe_alloc(struct bch_fs *c, struct ec_stripe_head *h) h->blocksize, h->disk_label); h->s = s; + h->nr_created++; return 0; } @@ -2278,9 +2279,10 @@ void bch2_new_stripes_to_text(struct printbuf *out, struct bch_fs *c) mutex_lock(&c->ec_stripe_head_lock); list_for_each_entry(h, &c->ec_stripe_head_list, list) { - prt_printf(out, "disk label %u algo %u redundancy %u %s:\n", + prt_printf(out, "disk label %u algo %u redundancy %u %s nr created %llu:\n", h->disk_label, h->algo, h->redundancy, - bch2_watermarks[h->watermark]); + bch2_watermarks[h->watermark], + h->nr_created); if (h->s) bch2_new_stripe_to_text(out, c, h->s); diff --git a/fs/bcachefs/ec.h b/fs/bcachefs/ec.h index b400c7e0ed9c..f02bb7d2e767 100644 --- a/fs/bcachefs/ec.h +++ b/fs/bcachefs/ec.h @@ -193,6 +193,8 @@ struct ec_stripe_head { unsigned redundancy; enum bch_watermark watermark; + u64 nr_created; + struct bch_devs_mask devs; unsigned nr_active_devs; From c9cabfb215d5ba6d972186f8eefea835e4697220 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 1 Sep 2024 16:45:34 -0400 Subject: [PATCH 293/302] bcachefs: improve bch2_new_stripe_to_text() also print out the new stripe key Signed-off-by: Kent Overstreet --- fs/bcachefs/ec.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index 24140c485eee..46667fce6b69 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -2270,6 +2270,8 @@ static void bch2_new_stripe_to_text(struct printbuf *out, struct bch_fs *c, for_each_set_bit(i, s->blocks_gotten, v->nr_blocks) prt_printf(out, " %u", s->blocks[i]); prt_newline(out); + bch2_bkey_val_to_text(out, c, bkey_i_to_s_c(&s->new_stripe.key)); + prt_newline(out); } void bch2_new_stripes_to_text(struct printbuf *out, struct bch_fs *c) From cb771fe891f97d847eda643553b31e99661f312d Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 1 Sep 2024 17:42:01 -0400 Subject: [PATCH 294/302] bcachefs: improve error message on too few devices for ec Signed-off-by: Kent Overstreet --- fs/bcachefs/ec.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index 46667fce6b69..cc76ae3d2073 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -1728,10 +1728,12 @@ ec_new_stripe_head_alloc(struct bch_fs *c, unsigned disk_label, rcu_read_lock(); h->devs = target_rw_devs(c, BCH_DATA_user, disk_label ? group_to_target(disk_label - 1) : 0); + unsigned nr_devs = dev_mask_nr(&h->devs); for_each_member_device_rcu(c, ca, &h->devs) if (!ca->mi.durability) __clear_bit(ca->dev_idx, h->devs.d); + unsigned nr_devs_with_durability = dev_mask_nr(&h->devs); h->blocksize = pick_blocksize(c, &h->devs); @@ -1745,9 +1747,20 @@ ec_new_stripe_head_alloc(struct bch_fs *c, unsigned disk_label, * If we only have redundancy + 1 devices, we're better off with just * replication: */ - if (h->nr_active_devs < h->redundancy + 2) - bch_err(c, "insufficient devices available to create stripe (have %u, need %u) - mismatched bucket sizes?", - h->nr_active_devs, h->redundancy + 2); + if (h->nr_active_devs < h->redundancy + 2) { + const char *err; + + if (nr_devs < h->redundancy + 2) + err = NULL; + else if (nr_devs_with_durability < h->redundancy + 2) + err = "cannot use durability=0 devices"; + else + err = "mismatched bucket sizes"; + + if (err) + bch_err(c, "insufficient devices available to create stripe (have %u, need %u): %s", + h->nr_active_devs, h->redundancy + 2, err); + } list_add(&h->list, &c->ec_stripe_head_list); return h; From 2aee59eb212266992c018f1ae2cf9c7f6c4ed1a9 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sat, 7 Sep 2024 16:31:47 -0400 Subject: [PATCH 295/302] bcachefs: improve error messages in bch2_ec_read_extent() Signed-off-by: Kent Overstreet --- fs/bcachefs/ec.c | 38 +++++++++++++++++++++----------------- fs/bcachefs/ec.h | 2 +- fs/bcachefs/io_read.c | 2 +- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index cc76ae3d2073..8af623b15ecf 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -829,13 +829,16 @@ err: } /* recovery read path: */ -int bch2_ec_read_extent(struct btree_trans *trans, struct bch_read_bio *rbio) +int bch2_ec_read_extent(struct btree_trans *trans, struct bch_read_bio *rbio, + struct bkey_s_c orig_k) { struct bch_fs *c = trans->c; - struct ec_stripe_buf *buf; + struct ec_stripe_buf *buf = NULL; struct closure cl; struct bch_stripe *v; unsigned i, offset; + const char *msg = NULL; + struct printbuf msgbuf = PRINTBUF; int ret = 0; closure_init_stack(&cl); @@ -848,32 +851,28 @@ int bch2_ec_read_extent(struct btree_trans *trans, struct bch_read_bio *rbio) ret = lockrestart_do(trans, get_stripe_key_trans(trans, rbio->pick.ec.idx, buf)); if (ret) { - bch_err_ratelimited(c, - "error doing reconstruct read: error %i looking up stripe", ret); - kfree(buf); - return -BCH_ERR_stripe_reconstruct; + msg = "stripe not found"; + goto err; } v = &bkey_i_to_stripe(&buf->key)->v; if (!bch2_ptr_matches_stripe(v, rbio->pick)) { - bch_err_ratelimited(c, - "error doing reconstruct read: pointer doesn't match stripe"); - ret = -BCH_ERR_stripe_reconstruct; + msg = "pointer doesn't match stripe"; goto err; } offset = rbio->bio.bi_iter.bi_sector - v->ptrs[rbio->pick.ec.block].offset; if (offset + bio_sectors(&rbio->bio) > le16_to_cpu(v->sectors)) { - bch_err_ratelimited(c, - "error doing reconstruct read: read is bigger than stripe"); - ret = -BCH_ERR_stripe_reconstruct; + msg = "read is bigger than stripe"; goto err; } ret = ec_stripe_buf_init(buf, offset, bio_sectors(&rbio->bio)); - if (ret) + if (ret) { + msg = "-ENOMEM"; goto err; + } for (i = 0; i < v->nr_blocks; i++) ec_block_io(c, buf, REQ_OP_READ, i, &cl); @@ -881,9 +880,7 @@ int bch2_ec_read_extent(struct btree_trans *trans, struct bch_read_bio *rbio) closure_sync(&cl); if (ec_nr_failed(buf) > v->nr_redundant) { - bch_err_ratelimited(c, - "error doing reconstruct read: unable to read enough blocks"); - ret = -BCH_ERR_stripe_reconstruct; + msg = "unable to read enough blocks"; goto err; } @@ -895,10 +892,17 @@ int bch2_ec_read_extent(struct btree_trans *trans, struct bch_read_bio *rbio) memcpy_to_bio(&rbio->bio, rbio->bio.bi_iter, buf->data[rbio->pick.ec.block] + ((offset - buf->offset) << 9)); -err: +out: ec_stripe_buf_exit(buf); kfree(buf); return ret; +err: + bch2_bkey_val_to_text(&msgbuf, c, orig_k); + bch_err_ratelimited(c, + "error doing reconstruct read: %s\n %s", msg, msgbuf.buf); + printbuf_exit(&msgbuf);; + ret = -BCH_ERR_stripe_reconstruct; + goto out; } /* stripe bucket accounting: */ diff --git a/fs/bcachefs/ec.h b/fs/bcachefs/ec.h index f02bb7d2e767..179a8674dac4 100644 --- a/fs/bcachefs/ec.h +++ b/fs/bcachefs/ec.h @@ -206,7 +206,7 @@ struct ec_stripe_head { struct ec_stripe_new *s; }; -int bch2_ec_read_extent(struct btree_trans *, struct bch_read_bio *); +int bch2_ec_read_extent(struct btree_trans *, struct bch_read_bio *, struct bkey_s_c); void *bch2_writepoint_ec_buf(struct bch_fs *, struct write_point *); diff --git a/fs/bcachefs/io_read.c b/fs/bcachefs/io_read.c index b3fba16b157b..b2f50e74bb76 100644 --- a/fs/bcachefs/io_read.c +++ b/fs/bcachefs/io_read.c @@ -1092,7 +1092,7 @@ get_bio: trans->notrace_relock_fail = true; } else { /* Attempting reconstruct read: */ - if (bch2_ec_read_extent(trans, rbio)) { + if (bch2_ec_read_extent(trans, rbio, k)) { bch2_rbio_error(rbio, READ_RETRY_AVOID, BLK_STS_IOERR); goto out; } From 934137b0c066de53c6df3191c922f6e371fa45a7 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sat, 7 Sep 2024 21:51:46 -0400 Subject: [PATCH 296/302] bcachefs: bch2_trigger_ptr() calculates sectors even when no device This is necessary for erasure coded pointers to devices that have been removed. Signed-off-by: Kent Overstreet --- fs/bcachefs/backpointers.h | 35 ++++++++++++++++++++++------------- fs/bcachefs/buckets.c | 8 +++++--- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/fs/bcachefs/backpointers.h b/fs/bcachefs/backpointers.h index 7daecadb764e..3b29fdf519dd 100644 --- a/fs/bcachefs/backpointers.h +++ b/fs/bcachefs/backpointers.h @@ -134,26 +134,35 @@ static inline enum bch_data_type bch2_bkey_ptr_data_type(struct bkey_s_c k, } } +static inline void __bch2_extent_ptr_to_bp(struct bch_fs *c, struct bch_dev *ca, + enum btree_id btree_id, unsigned level, + struct bkey_s_c k, struct extent_ptr_decoded p, + const union bch_extent_entry *entry, + struct bpos *bucket_pos, struct bch_backpointer *bp, + u64 sectors) +{ + u32 bucket_offset; + *bucket_pos = PTR_BUCKET_POS_OFFSET(ca, &p.ptr, &bucket_offset); + *bp = (struct bch_backpointer) { + .btree_id = btree_id, + .level = level, + .data_type = bch2_bkey_ptr_data_type(k, p, entry), + .bucket_offset = ((u64) bucket_offset << MAX_EXTENT_COMPRESS_RATIO_SHIFT) + + p.crc.offset, + .bucket_len = sectors, + .pos = k.k->p, + }; +} + static inline void bch2_extent_ptr_to_bp(struct bch_fs *c, struct bch_dev *ca, enum btree_id btree_id, unsigned level, struct bkey_s_c k, struct extent_ptr_decoded p, const union bch_extent_entry *entry, struct bpos *bucket_pos, struct bch_backpointer *bp) { - enum bch_data_type data_type = bch2_bkey_ptr_data_type(k, p, entry); - s64 sectors = level ? btree_sectors(c) : k.k->size; - u32 bucket_offset; + u64 sectors = ptr_disk_sectors(level ? btree_sectors(c) : k.k->size, p); - *bucket_pos = PTR_BUCKET_POS_OFFSET(ca, &p.ptr, &bucket_offset); - *bp = (struct bch_backpointer) { - .btree_id = btree_id, - .level = level, - .data_type = data_type, - .bucket_offset = ((u64) bucket_offset << MAX_EXTENT_COMPRESS_RATIO_SHIFT) + - p.crc.offset, - .bucket_len = ptr_disk_sectors(sectors, p), - .pos = k.k->p, - }; + __bch2_extent_ptr_to_bp(c, ca, btree_id, level, k, p, entry, bucket_pos, bp, sectors); } int bch2_get_next_backpointer(struct btree_trans *, struct bch_dev *ca, struct bpos, int, diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c index 0732ac8f682d..546cd01a72e3 100644 --- a/fs/bcachefs/buckets.c +++ b/fs/bcachefs/buckets.c @@ -565,11 +565,14 @@ static int bch2_trigger_pointer(struct btree_trans *trans, s64 *sectors, enum btree_iter_update_trigger_flags flags) { + struct bch_fs *c = trans->c; bool insert = !(flags & BTREE_TRIGGER_overwrite); struct printbuf buf = PRINTBUF; int ret = 0; - struct bch_fs *c = trans->c; + u64 abs_sectors = ptr_disk_sectors(level ? btree_sectors(c) : k.k->size, p); + *sectors = insert ? abs_sectors : -abs_sectors; + struct bch_dev *ca = bch2_dev_tryget(c, p.ptr.dev); if (unlikely(!ca)) { if (insert && p.ptr.dev != BCH_SB_MEMBER_INVALID) @@ -579,8 +582,7 @@ static int bch2_trigger_pointer(struct btree_trans *trans, struct bpos bucket; struct bch_backpointer bp; - bch2_extent_ptr_to_bp(trans->c, ca, btree_id, level, k, p, entry, &bucket, &bp); - *sectors = insert ? bp.bucket_len : -((s64) bp.bucket_len); + __bch2_extent_ptr_to_bp(trans->c, ca, btree_id, level, k, p, entry, &bucket, &bp, abs_sectors); if (flags & BTREE_TRIGGER_transactional) { struct bkey_i_alloc_v4 *a = bch2_trans_start_alloc_update(trans, bucket, 0); From ad8d1f77fc425f42d1b297486eea36864ae82c41 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 1 Sep 2024 18:35:52 -0400 Subject: [PATCH 297/302] bcachefs: bch2_dev_remove_stripes() We can now correctly force-remove a device that has stripes on it; this uses the new BCH_SB_MEMBER_INVALID sentinal value. Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_background.c | 7 ++-- fs/bcachefs/ec.c | 67 ++++++++++++++++++++++++++++++++++ fs/bcachefs/ec.h | 2 + fs/bcachefs/errcode.h | 1 + 4 files changed, 74 insertions(+), 3 deletions(-) diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c index d4f0b62e3138..87ce65e46304 100644 --- a/fs/bcachefs/alloc_background.c +++ b/fs/bcachefs/alloc_background.c @@ -2310,7 +2310,8 @@ int bch2_dev_remove_alloc(struct bch_fs *c, struct bch_dev *ca) * We clear the LRU and need_discard btrees first so that we don't race * with bch2_do_invalidates() and bch2_do_discards() */ - ret = bch2_btree_delete_range(c, BTREE_ID_lru, start, end, + ret = bch2_dev_remove_stripes(c, ca->dev_idx) ?: + bch2_btree_delete_range(c, BTREE_ID_lru, start, end, BTREE_TRIGGER_norun, NULL) ?: bch2_btree_delete_range(c, BTREE_ID_need_discard, start, end, BTREE_TRIGGER_norun, NULL) ?: @@ -2318,10 +2319,10 @@ int bch2_dev_remove_alloc(struct bch_fs *c, struct bch_dev *ca) BTREE_TRIGGER_norun, NULL) ?: bch2_btree_delete_range(c, BTREE_ID_backpointers, start, end, BTREE_TRIGGER_norun, NULL) ?: - bch2_btree_delete_range(c, BTREE_ID_alloc, start, end, - BTREE_TRIGGER_norun, NULL) ?: bch2_btree_delete_range(c, BTREE_ID_bucket_gens, start, end, BTREE_TRIGGER_norun, NULL) ?: + bch2_btree_delete_range(c, BTREE_ID_alloc, start, end, + BTREE_TRIGGER_norun, NULL) ?: bch2_dev_usage_remove(c, ca->dev_idx); bch_err_msg(ca, ret, "removing dev alloc info"); return ret; diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index 8af623b15ecf..f58c3e78ea45 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -2169,6 +2169,73 @@ err: return ERR_PTR(ret); } +/* device removal */ + +static int bch2_invalidate_stripe_to_dev(struct btree_trans *trans, struct bkey_s_c k_a) +{ + struct bch_alloc_v4 a_convert; + const struct bch_alloc_v4 *a = bch2_alloc_to_v4(k_a, &a_convert); + + if (!a->stripe) + return 0; + + if (a->stripe_sectors) { + bch_err(trans->c, "trying to invalidate device in stripe when bucket has stripe data"); + return -BCH_ERR_invalidate_stripe_to_dev; + } + + struct btree_iter iter; + struct bkey_i_stripe *s = + bch2_bkey_get_mut_typed(trans, &iter, BTREE_ID_stripes, POS(0, a->stripe), + BTREE_ITER_slots, stripe); + int ret = PTR_ERR_OR_ZERO(s); + if (ret) + return ret; + + struct disk_accounting_pos acc = { + .type = BCH_DISK_ACCOUNTING_replicas, + }; + + s64 sectors = 0; + for (unsigned i = 0; i < s->v.nr_blocks; i++) + sectors -= stripe_blockcount_get(&s->v, i); + + bch2_bkey_to_replicas(&acc.replicas, bkey_i_to_s_c(&s->k_i)); + acc.replicas.data_type = BCH_DATA_user; + ret = bch2_disk_accounting_mod(trans, &acc, §ors, 1, false); + if (ret) + goto err; + + struct bkey_ptrs ptrs = bch2_bkey_ptrs(bkey_i_to_s(&s->k_i)); + bkey_for_each_ptr(ptrs, ptr) + if (ptr->dev == k_a.k->p.inode) + ptr->dev = BCH_SB_MEMBER_INVALID; + + sectors = -sectors; + + bch2_bkey_to_replicas(&acc.replicas, bkey_i_to_s_c(&s->k_i)); + acc.replicas.data_type = BCH_DATA_user; + ret = bch2_disk_accounting_mod(trans, &acc, §ors, 1, false); + if (ret) + goto err; +err: + bch2_trans_iter_exit(trans, &iter); + return ret; +} + +int bch2_dev_remove_stripes(struct bch_fs *c, unsigned dev_idx) +{ + return bch2_trans_run(c, + for_each_btree_key_upto_commit(trans, iter, + BTREE_ID_alloc, POS(dev_idx, 0), POS(dev_idx, U64_MAX), + BTREE_ITER_intent, k, + NULL, NULL, 0, ({ + bch2_invalidate_stripe_to_dev(trans, k); + }))); +} + +/* startup/shutdown */ + static void __bch2_ec_stop(struct bch_fs *c, struct bch_dev *ca) { struct ec_stripe_head *h; diff --git a/fs/bcachefs/ec.h b/fs/bcachefs/ec.h index 179a8674dac4..c432040238cd 100644 --- a/fs/bcachefs/ec.h +++ b/fs/bcachefs/ec.h @@ -251,6 +251,8 @@ static inline void ec_stripe_new_put(struct bch_fs *c, struct ec_stripe_new *s, } } +int bch2_dev_remove_stripes(struct bch_fs *, unsigned); + void bch2_ec_stop_dev(struct bch_fs *, struct bch_dev *); void bch2_fs_ec_stop(struct bch_fs *); void bch2_fs_ec_flush(struct bch_fs *); diff --git a/fs/bcachefs/errcode.h b/fs/bcachefs/errcode.h index 5241910c1cb0..60b7875adada 100644 --- a/fs/bcachefs/errcode.h +++ b/fs/bcachefs/errcode.h @@ -253,6 +253,7 @@ x(EIO, key_type_error) \ x(EIO, no_device_to_read_from) \ x(EIO, missing_indirect_extent) \ + x(EIO, invalidate_stripe_to_dev) \ x(BCH_ERR_btree_node_read_err, btree_node_read_err_fixable) \ x(BCH_ERR_btree_node_read_err, btree_node_read_err_want_retry) \ x(BCH_ERR_btree_node_read_err, btree_node_read_err_must_retry) \ From 83ccd9b31d5356f9943b8fd0f996ec9fefcb25f1 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Fri, 6 Sep 2024 19:12:53 -0400 Subject: [PATCH 298/302] bcachefs: bch_fs.rw_devs_change_count Add a counter that's incremented whenever rw devices change; this will be used for erasure coding so that it can keep ec_stripe_head in sync and not deadlock on a new stripe when a device it wants goes away. Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_background.c | 12 ++++++++---- fs/bcachefs/bcachefs.h | 1 + 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c index 87ce65e46304..645b5ed4babb 100644 --- a/fs/bcachefs/alloc_background.c +++ b/fs/bcachefs/alloc_background.c @@ -2463,13 +2463,15 @@ static bool bch2_dev_has_open_write_point(struct bch_fs *c, struct bch_dev *ca) /* device goes ro: */ void bch2_dev_allocator_remove(struct bch_fs *c, struct bch_dev *ca) { - unsigned i; + lockdep_assert_held(&c->state_lock); /* First, remove device from allocation groups: */ - for (i = 0; i < ARRAY_SIZE(c->rw_devs); i++) + for (unsigned i = 0; i < ARRAY_SIZE(c->rw_devs); i++) clear_bit(ca->dev_idx, c->rw_devs[i].d); + c->rw_devs_change_count++; + /* * Capacity is calculated based off of devices in allocation groups: */ @@ -2498,11 +2500,13 @@ void bch2_dev_allocator_remove(struct bch_fs *c, struct bch_dev *ca) /* device goes rw: */ void bch2_dev_allocator_add(struct bch_fs *c, struct bch_dev *ca) { - unsigned i; + lockdep_assert_held(&c->state_lock); - for (i = 0; i < ARRAY_SIZE(c->rw_devs); i++) + for (unsigned i = 0; i < ARRAY_SIZE(c->rw_devs); i++) if (ca->mi.data_allowed & (1 << i)) set_bit(ca->dev_idx, c->rw_devs[i].d); + + c->rw_devs_change_count++; } void bch2_dev_allocator_background_exit(struct bch_dev *ca) diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h index b2475b9555ac..c711d4c27a03 100644 --- a/fs/bcachefs/bcachefs.h +++ b/fs/bcachefs/bcachefs.h @@ -871,6 +871,7 @@ struct bch_fs { /* ALLOCATION */ struct bch_devs_mask rw_devs[BCH_DATA_NR]; + unsigned long rw_devs_change_count; u64 capacity; /* sectors */ u64 reserved; /* sectors */ From 035d72f72c9172a29bba4e09620d286ed8496356 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Fri, 6 Sep 2024 19:14:36 -0400 Subject: [PATCH 299/302] bcachefs: bch2_ec_stripe_head_get() now checks for change in rw devices This factors out ec_strie_head_devs_update(), which initializes the bitmap of devices we're allocating from, and runs it every time c->rw_devs_change_count changes. We also cancel pending, not allocated stripes, since they may refer to devices that are no longer available. Signed-off-by: Kent Overstreet --- fs/bcachefs/ec.c | 118 +++++++++++++++++++++++++++++------------------ fs/bcachefs/ec.h | 3 ++ 2 files changed, 77 insertions(+), 44 deletions(-) diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index f58c3e78ea45..f167ea454ded 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -1572,10 +1572,12 @@ void bch2_ec_do_stripe_creates(struct bch_fs *c) bch2_write_ref_put(c, BCH_WRITE_REF_stripe_create); } -static void ec_stripe_set_pending(struct bch_fs *c, struct ec_stripe_head *h) +static void ec_stripe_new_set_pending(struct bch_fs *c, struct ec_stripe_head *h) { struct ec_stripe_new *s = h->s; + lockdep_assert_held(&h->lock); + BUG_ON(!s->allocated && !s->err); h->s = NULL; @@ -1588,6 +1590,12 @@ static void ec_stripe_set_pending(struct bch_fs *c, struct ec_stripe_head *h) ec_stripe_new_put(c, s, STRIPE_REF_io); } +static void ec_stripe_new_cancel(struct bch_fs *c, struct ec_stripe_head *h, int err) +{ + h->s->err = err; + ec_stripe_new_set_pending(c, h); +} + void bch2_ec_bucket_cancel(struct bch_fs *c, struct open_bucket *ob) { struct ec_stripe_new *s = ob->ec; @@ -1711,6 +1719,60 @@ static int ec_new_stripe_alloc(struct bch_fs *c, struct ec_stripe_head *h) return 0; } +static void ec_stripe_head_devs_update(struct bch_fs *c, struct ec_stripe_head *h) +{ + struct bch_devs_mask devs = h->devs; + + rcu_read_lock(); + h->devs = target_rw_devs(c, BCH_DATA_user, h->disk_label + ? group_to_target(h->disk_label - 1) + : 0); + unsigned nr_devs = dev_mask_nr(&h->devs); + + for_each_member_device_rcu(c, ca, &h->devs) + if (!ca->mi.durability) + __clear_bit(ca->dev_idx, h->devs.d); + unsigned nr_devs_with_durability = dev_mask_nr(&h->devs); + + h->blocksize = pick_blocksize(c, &h->devs); + + h->nr_active_devs = 0; + for_each_member_device_rcu(c, ca, &h->devs) + if (ca->mi.bucket_size == h->blocksize) + h->nr_active_devs++; + + rcu_read_unlock(); + + /* + * If we only have redundancy + 1 devices, we're better off with just + * replication: + */ + h->insufficient_devs = h->nr_active_devs < h->redundancy + 2; + + if (h->insufficient_devs) { + const char *err; + + if (nr_devs < h->redundancy + 2) + err = NULL; + else if (nr_devs_with_durability < h->redundancy + 2) + err = "cannot use durability=0 devices"; + else + err = "mismatched bucket sizes"; + + if (err) + bch_err(c, "insufficient devices available to create stripe (have %u, need %u): %s", + h->nr_active_devs, h->redundancy + 2, err); + } + + struct bch_devs_mask devs_leaving; + bitmap_andnot(devs_leaving.d, devs.d, h->devs.d, BCH_SB_MEMBERS_MAX); + + if (h->s && !h->s->allocated && dev_mask_nr(&devs_leaving)) + ec_stripe_new_cancel(c, h, -EINTR); + + h->rw_devs_change_count = c->rw_devs_change_count; +} + static struct ec_stripe_head * ec_new_stripe_head_alloc(struct bch_fs *c, unsigned disk_label, unsigned algo, unsigned redundancy, @@ -1730,42 +1792,6 @@ ec_new_stripe_head_alloc(struct bch_fs *c, unsigned disk_label, h->redundancy = redundancy; h->watermark = watermark; - rcu_read_lock(); - h->devs = target_rw_devs(c, BCH_DATA_user, disk_label ? group_to_target(disk_label - 1) : 0); - unsigned nr_devs = dev_mask_nr(&h->devs); - - for_each_member_device_rcu(c, ca, &h->devs) - if (!ca->mi.durability) - __clear_bit(ca->dev_idx, h->devs.d); - unsigned nr_devs_with_durability = dev_mask_nr(&h->devs); - - h->blocksize = pick_blocksize(c, &h->devs); - - for_each_member_device_rcu(c, ca, &h->devs) - if (ca->mi.bucket_size == h->blocksize) - h->nr_active_devs++; - - rcu_read_unlock(); - - /* - * If we only have redundancy + 1 devices, we're better off with just - * replication: - */ - if (h->nr_active_devs < h->redundancy + 2) { - const char *err; - - if (nr_devs < h->redundancy + 2) - err = NULL; - else if (nr_devs_with_durability < h->redundancy + 2) - err = "cannot use durability=0 devices"; - else - err = "mismatched bucket sizes"; - - if (err) - bch_err(c, "insufficient devices available to create stripe (have %u, need %u): %s", - h->nr_active_devs, h->redundancy + 2, err); - } - list_add(&h->list, &c->ec_stripe_head_list); return h; } @@ -1776,7 +1802,7 @@ void bch2_ec_stripe_head_put(struct bch_fs *c, struct ec_stripe_head *h) h->s->allocated && bitmap_weight(h->s->blocks_allocated, h->s->nr_data) == h->s->nr_data) - ec_stripe_set_pending(c, h); + ec_stripe_new_set_pending(c, h); mutex_unlock(&h->lock); } @@ -1801,7 +1827,7 @@ __bch2_ec_stripe_head_get(struct btree_trans *trans, if (test_bit(BCH_FS_going_ro, &c->flags)) { h = ERR_PTR(-BCH_ERR_erofs_no_writes); - goto found; + goto err; } list_for_each_entry(h, &c->ec_stripe_head_list, list) @@ -1810,18 +1836,23 @@ __bch2_ec_stripe_head_get(struct btree_trans *trans, h->redundancy == redundancy && h->watermark == watermark) { ret = bch2_trans_mutex_lock(trans, &h->lock); - if (ret) + if (ret) { h = ERR_PTR(ret); + goto err; + } goto found; } h = ec_new_stripe_head_alloc(c, disk_label, algo, redundancy, watermark); found: - if (!IS_ERR_OR_NULL(h) && - h->nr_active_devs < h->redundancy + 2) { + if (h->rw_devs_change_count != c->rw_devs_change_count) + ec_stripe_head_devs_update(c, h); + + if (h->insufficient_devs) { mutex_unlock(&h->lock); h = NULL; } +err: mutex_unlock(&c->ec_stripe_head_lock); return h; } @@ -2261,8 +2292,7 @@ static void __bch2_ec_stop(struct bch_fs *c, struct bch_dev *ca) } goto unlock; found: - h->s->err = -BCH_ERR_erofs_no_writes; - ec_stripe_set_pending(c, h); + ec_stripe_new_cancel(c, h, -BCH_ERR_erofs_no_writes); unlock: mutex_unlock(&h->lock); } diff --git a/fs/bcachefs/ec.h b/fs/bcachefs/ec.h index c432040238cd..43326370b410 100644 --- a/fs/bcachefs/ec.h +++ b/fs/bcachefs/ec.h @@ -192,6 +192,9 @@ struct ec_stripe_head { unsigned algo; unsigned redundancy; enum bch_watermark watermark; + bool insufficient_devs; + + unsigned long rw_devs_change_count; u64 nr_created; From d5c5b337f8da6c9eebbe6cf9dfbd38fd9503eb35 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Fri, 6 Sep 2024 20:22:26 -0400 Subject: [PATCH 300/302] bcachefs: Don't drop devices with stripe pointers Signed-off-by: Kent Overstreet --- fs/bcachefs/data_update.c | 2 +- fs/bcachefs/ec.c | 2 +- fs/bcachefs/extents.c | 17 +++++++++++++---- fs/bcachefs/extents.h | 20 +++++++++++++++++--- 4 files changed, 32 insertions(+), 9 deletions(-) diff --git a/fs/bcachefs/data_update.c b/fs/bcachefs/data_update.c index 004894ad4147..757b9884ef55 100644 --- a/fs/bcachefs/data_update.c +++ b/fs/bcachefs/data_update.c @@ -571,7 +571,7 @@ int bch2_extent_drop_ptrs(struct btree_trans *trans, while (data_opts.kill_ptrs) { unsigned i = 0, drop = __fls(data_opts.kill_ptrs); - bch2_bkey_drop_ptrs(bkey_i_to_s(n), ptr, i++ == drop); + bch2_bkey_drop_ptrs_noerror(bkey_i_to_s(n), ptr, i++ == drop); data_opts.kill_ptrs ^= 1U << drop; } diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index f167ea454ded..1587c6e1866a 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -1322,7 +1322,7 @@ static int ec_stripe_update_extent(struct btree_trans *trans, bkey_reassemble(n, k); - bch2_bkey_drop_ptrs(bkey_i_to_s(n), ptr, ptr->dev != dev); + bch2_bkey_drop_ptrs_noerror(bkey_i_to_s(n), ptr, ptr->dev != dev); ec_ptr = bch2_bkey_has_device(bkey_i_to_s(n), dev); BUG_ON(!ec_ptr); diff --git a/fs/bcachefs/extents.c b/fs/bcachefs/extents.c index d370d679e015..cc0d22085aef 100644 --- a/fs/bcachefs/extents.c +++ b/fs/bcachefs/extents.c @@ -818,6 +818,18 @@ void bch2_bkey_drop_ptr_noerror(struct bkey_s k, struct bch_extent_ptr *ptr) void bch2_bkey_drop_ptr(struct bkey_s k, struct bch_extent_ptr *ptr) { + if (k.k->type != KEY_TYPE_stripe) { + struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k.s_c); + const union bch_extent_entry *entry; + struct extent_ptr_decoded p; + + bkey_for_each_ptr_decode(k.k, ptrs, p, entry) + if (p.ptr.dev == ptr->dev && p.has_ec) { + ptr->dev = BCH_SB_MEMBER_INVALID; + return; + } + } + bool have_dirty = bch2_bkey_dirty_devs(k.s_c).nr; bch2_bkey_drop_ptr_noerror(k, ptr); @@ -845,10 +857,7 @@ void bch2_bkey_drop_device(struct bkey_s k, unsigned dev) void bch2_bkey_drop_device_noerror(struct bkey_s k, unsigned dev) { - struct bch_extent_ptr *ptr = bch2_bkey_has_device(k, dev); - - if (ptr) - bch2_bkey_drop_ptr_noerror(k, ptr); + bch2_bkey_drop_ptrs_noerror(k, ptr, ptr->dev == dev); } const struct bch_extent_ptr *bch2_bkey_has_device_c(struct bkey_s_c k, unsigned dev) diff --git a/fs/bcachefs/extents.h b/fs/bcachefs/extents.h index 412e1c5b05a0..ed5001dd662e 100644 --- a/fs/bcachefs/extents.h +++ b/fs/bcachefs/extents.h @@ -611,9 +611,6 @@ unsigned bch2_extent_ptr_desired_durability(struct bch_fs *, struct extent_ptr_d unsigned bch2_extent_ptr_durability(struct bch_fs *, struct extent_ptr_decoded *); unsigned bch2_bkey_durability(struct bch_fs *, struct bkey_s_c); -void bch2_bkey_drop_device(struct bkey_s, unsigned); -void bch2_bkey_drop_device_noerror(struct bkey_s, unsigned); - const struct bch_extent_ptr *bch2_bkey_has_device_c(struct bkey_s_c, unsigned); static inline struct bch_extent_ptr *bch2_bkey_has_device(struct bkey_s k, unsigned dev) @@ -652,6 +649,23 @@ void bch2_extent_ptr_decoded_append(struct bkey_i *, void bch2_bkey_drop_ptr_noerror(struct bkey_s, struct bch_extent_ptr *); void bch2_bkey_drop_ptr(struct bkey_s, struct bch_extent_ptr *); +void bch2_bkey_drop_device_noerror(struct bkey_s, unsigned); +void bch2_bkey_drop_device(struct bkey_s, unsigned); + +#define bch2_bkey_drop_ptrs_noerror(_k, _ptr, _cond) \ +do { \ + __label__ _again; \ + struct bkey_ptrs _ptrs; \ +_again: \ + _ptrs = bch2_bkey_ptrs(_k); \ + \ + bkey_for_each_ptr(_ptrs, _ptr) \ + if (_cond) { \ + bch2_bkey_drop_ptr_noerror(_k, _ptr); \ + goto _again; \ + } \ +} while (0) + #define bch2_bkey_drop_ptrs(_k, _ptr, _cond) \ do { \ __label__ _again; \ From abb43dd677f3c5508dc369a61f82f89a8b16b811 Mon Sep 17 00:00:00 2001 From: Yang Li Date: Mon, 9 Sep 2024 08:58:02 +0800 Subject: [PATCH 301/302] bcachefs: Remove duplicated include in backpointers.c The header files bbpos.h is included twice in backpointers.c, so one inclusion of each can be removed. Reported-by: Abaci Robot Closes: https://bugzilla.openanolis.cn/show_bug.cgi?id=10783 Signed-off-by: Yang Li Signed-off-by: Kent Overstreet --- fs/bcachefs/backpointers.c | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/bcachefs/backpointers.c b/fs/bcachefs/backpointers.c index b88d1801652c..e11989a57ca0 100644 --- a/fs/bcachefs/backpointers.c +++ b/fs/bcachefs/backpointers.c @@ -3,7 +3,6 @@ #include "bbpos.h" #include "alloc_background.h" #include "backpointers.h" -#include "bbpos.h" #include "bkey_buf.h" #include "btree_cache.h" #include "btree_update.h" From 025c55a4c7f11ea38521c6e797f3192ad8768c93 Mon Sep 17 00:00:00 2001 From: Diogo Jahchan Koike Date: Tue, 10 Sep 2024 18:18:34 -0300 Subject: [PATCH 302/302] bcachefs: return err ptr instead of null in read sb clean syzbot reported a null-ptr-deref in bch2_fs_start. [0] When a sb is marked clear but doesn't have a clean section bch2_read_superblock_clean returns NULL which PTR_ERR_OR_ZERO lets through, eventually leading to a null ptr dereference down the line. Adjust read sb clean to return an ERR_PTR indicating the invalid clean section. [0] https://syzkaller.appspot.com/bug?extid=1cecc37d87c4286e5543 Reported-by: syzbot+1cecc37d87c4286e5543@syzkaller.appspotmail.com Closes: https://syzkaller.appspot.com/bug?extid=1cecc37d87c4286e5543 Signed-off-by: Diogo Jahchan Koike Signed-off-by: Kent Overstreet --- fs/bcachefs/sb-clean.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/bcachefs/sb-clean.c b/fs/bcachefs/sb-clean.c index c57d42bb8d1b..025848a9c4c0 100644 --- a/fs/bcachefs/sb-clean.c +++ b/fs/bcachefs/sb-clean.c @@ -155,7 +155,7 @@ struct bch_sb_field_clean *bch2_read_superblock_clean(struct bch_fs *c) SET_BCH_SB_CLEAN(c->disk_sb.sb, false); c->sb.clean = false; mutex_unlock(&c->sb_lock); - return NULL; + return ERR_PTR(-BCH_ERR_invalid_sb_clean); } clean = kmemdup(sb_clean, vstruct_bytes(&sb_clean->field),