diff --git a/Documentation/admin-guide/pm/intel_idle.rst b/Documentation/admin-guide/pm/intel_idle.rst index ed6f055d4b14..188d52cd26e8 100644 --- a/Documentation/admin-guide/pm/intel_idle.rst +++ b/Documentation/admin-guide/pm/intel_idle.rst @@ -260,6 +260,17 @@ mode to off when the CPU is in any one of the available idle states. This may help performance of a sibling CPU at the expense of a slightly higher wakeup latency for the idle CPU. +The ``table`` argument allows customization of idle state latency and target +residency. The syntax is a comma-separated list of ``name:latency:residency`` +entries, where ``name`` is the idle state name, ``latency`` is the exit latency +in microseconds, and ``residency`` is the target residency in microseconds. It +is not necessary to specify all idle states; only those to be customized. For +example, ``C1:1:3,C6:50:100`` sets the exit latency and target residency for +C1 and C6 to 1/3 and 50/100 microseconds, respectively. Remaining idle states +keep their default values. The driver verifies that deeper idle states have +higher latency and target residency than shallower ones. Also, target +residency cannot be smaller than exit latency. If any of these conditions is +not met, the driver ignores the entire ``table`` parameter. .. _intel-idle-core-and-package-idle-states: diff --git a/Documentation/filesystems/overlayfs.rst b/Documentation/filesystems/overlayfs.rst index 1ddbb4f3005e..33fa170282f7 100644 --- a/Documentation/filesystems/overlayfs.rst +++ b/Documentation/filesystems/overlayfs.rst @@ -753,9 +753,9 @@ Note: the mount options index=off,nfs_export=on are conflicting for a read-write mount and will result in an error. Note: the mount option uuid=off can be used to replace UUID of the underlying -filesystem in file handles with null, and effectively disable UUID checks. This +filesystem in file handles with null, in order to relax the UUID checks. This can be useful in case the underlying disk is copied and the UUID of this copy -is changed. This is only applicable if all lower/upper/work directories are on +is changed. This is only applicable if all lower directories are on the same filesystem, otherwise it will fallback to normal behaviour. @@ -769,7 +769,7 @@ controlled by the "uuid" mount option, which supports these values: UUID of overlayfs is null. fsid is taken from upper most filesystem. - "off": UUID of overlayfs is null. fsid is taken from upper most filesystem. - UUID of underlying layers is ignored. + UUID of underlying layers is ignored and null used instead. - "on": UUID of overlayfs is generated and used to report a unique fsid. UUID is stored in xattr "trusted.overlay.uuid", making overlayfs fsid diff --git a/Makefile b/Makefile index 944b472894a2..93b27d6b9365 100644 --- a/Makefile +++ b/Makefile @@ -1525,6 +1525,15 @@ ifneq ($(wildcard $(resolve_btfids_O)),) $(Q)$(MAKE) -sC $(srctree)/tools/bpf/resolve_btfids O=$(resolve_btfids_O) clean endif +PHONY += objtool_clean + +objtool_O = $(abspath $(objtree))/tools/objtool + +objtool_clean: +ifneq ($(wildcard $(objtool_O)),) + $(Q)$(MAKE) -sC $(abs_srctree)/tools/objtool O=$(objtool_O) srctree=$(abs_srctree) clean +endif + tools/: FORCE $(Q)mkdir -p $(objtree)/tools $(Q)$(MAKE) O=$(abspath $(objtree)) subdir=tools -C $(srctree)/tools/ @@ -1696,7 +1705,7 @@ vmlinuxclean: $(Q)$(CONFIG_SHELL) $(srctree)/scripts/link-vmlinux.sh clean $(Q)$(if $(ARCH_POSTLINK), $(MAKE) -f $(ARCH_POSTLINK) clean) -clean: archclean vmlinuxclean resolve_btfids_clean +clean: archclean vmlinuxclean resolve_btfids_clean objtool_clean # mrproper - Delete all generated files, including .config # diff --git a/drivers/acpi/acpi_video.c b/drivers/acpi/acpi_video.c index 69469757b965..3d6e7306f29a 100644 --- a/drivers/acpi/acpi_video.c +++ b/drivers/acpi/acpi_video.c @@ -2116,6 +2116,7 @@ static void acpi_video_bus_remove(struct platform_device *pdev) kfree(video->attached_array); kfree(video); + device->driver_data = NULL; } static int __init is_i740(struct pci_dev *dev) diff --git a/drivers/acpi/battery.c b/drivers/acpi/battery.c index 3bbddd6f622c..8fbad8bc4650 100644 --- a/drivers/acpi/battery.c +++ b/drivers/acpi/battery.c @@ -1066,9 +1066,6 @@ static void acpi_battery_notify(acpi_handle handle, u32 event, void *data) struct acpi_device *device = battery->device; struct power_supply *old; - if (!battery) - return; - guard(mutex)(&battery->update_lock); old = battery->bat; @@ -1274,13 +1271,9 @@ fail: static void acpi_battery_remove(struct platform_device *pdev) { - struct acpi_device *device = ACPI_COMPANION(&pdev->dev); struct acpi_battery *battery = platform_get_drvdata(pdev); - if (!device || !battery) - return; - - acpi_dev_remove_notify_handler(device, ACPI_ALL_NOTIFY, + acpi_dev_remove_notify_handler(battery->device, ACPI_ALL_NOTIFY, acpi_battery_notify); device_init_wakeup(&pdev->dev, false); diff --git a/drivers/acpi/button.c b/drivers/acpi/button.c index b899b8745fed..7a06194d078b 100644 --- a/drivers/acpi/button.c +++ b/drivers/acpi/button.c @@ -170,7 +170,7 @@ static struct platform_driver acpi_button_driver = { struct acpi_button { struct acpi_device *adev; - struct platform_device *pdev; + struct device *dev; /* physical button device */ unsigned int type; struct input_dev *input; char phys[32]; /* for input device */ @@ -398,7 +398,7 @@ static int acpi_lid_update_state(struct acpi_button *button, return state; if (state && signal_wakeup) - acpi_pm_wakeup_event(&button->pdev->dev); + acpi_pm_wakeup_event(button->dev); return acpi_lid_notify_state(button, state); } @@ -455,7 +455,7 @@ static void acpi_button_notify(acpi_handle handle, u32 event, void *data) return; } - acpi_pm_wakeup_event(&button->pdev->dev); + acpi_pm_wakeup_event(button->dev); if (button->suspended || event == ACPI_BUTTON_NOTIFY_WAKE) return; @@ -550,7 +550,7 @@ static int acpi_button_probe(struct platform_device *pdev) platform_set_drvdata(pdev, button); - button->pdev = pdev; + button->dev = &pdev->dev; button->adev = device; button->input = input = input_allocate_device(); if (!input) { @@ -625,6 +625,8 @@ static int acpi_button_probe(struct platform_device *pdev) goto err_remove_fs; } + device_init_wakeup(button->dev, true); + switch (device->device_type) { case ACPI_BUS_TYPE_POWER_BUTTON: status = acpi_install_fixed_event_handler(ACPI_EVENT_POWER_BUTTON, @@ -655,11 +657,11 @@ static int acpi_button_probe(struct platform_device *pdev) lid_device = device; } - device_init_wakeup(&pdev->dev, true); pr_info("%s [%s]\n", name, acpi_device_bid(device)); return 0; err_input_unregister: + device_init_wakeup(button->dev, false); input_unregister_device(input); err_remove_fs: acpi_button_remove_fs(button); @@ -670,10 +672,10 @@ err_free_button: static void acpi_button_remove(struct platform_device *pdev) { - struct acpi_device *device = ACPI_COMPANION(&pdev->dev); struct acpi_button *button = platform_get_drvdata(pdev); + struct acpi_device *adev = button->adev; - switch (device->device_type) { + switch (adev->device_type) { case ACPI_BUS_TYPE_POWER_BUTTON: acpi_remove_fixed_event_handler(ACPI_EVENT_POWER_BUTTON, acpi_button_event); @@ -683,7 +685,7 @@ static void acpi_button_remove(struct platform_device *pdev) acpi_button_event); break; default: - acpi_remove_notify_handler(device->handle, ACPI_DEVICE_NOTIFY, + acpi_remove_notify_handler(adev->handle, ACPI_DEVICE_NOTIFY, button->type == ACPI_BUTTON_TYPE_LID ? acpi_lid_notify : acpi_button_notify); @@ -691,6 +693,8 @@ static void acpi_button_remove(struct platform_device *pdev) } acpi_os_wait_events_complete(); + device_init_wakeup(button->dev, false); + acpi_button_remove_fs(button); input_unregister_device(button->input); kfree(button); diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index a09bdabaa804..0e6ffb188fe7 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -362,7 +362,7 @@ static int send_pcc_cmd(int pcc_ss_id, u16 cmd) end: if (cmd == CMD_WRITE) { if (unlikely(ret)) { - for_each_possible_cpu(i) { + for_each_online_cpu(i) { struct cpc_desc *desc = per_cpu(cpc_desc_ptr, i); if (!desc) @@ -524,7 +524,7 @@ int acpi_get_psd_map(unsigned int cpu, struct cppc_cpudata *cpu_data) else if (pdomain->coord_type == DOMAIN_COORD_TYPE_SW_ANY) cpu_data->shared_type = CPUFREQ_SHARED_TYPE_ANY; - for_each_possible_cpu(i) { + for_each_online_cpu(i) { if (i == cpu) continue; diff --git a/drivers/acpi/ec.c b/drivers/acpi/ec.c index 197970339edc..f7edc664e064 100644 --- a/drivers/acpi/ec.c +++ b/drivers/acpi/ec.c @@ -1754,12 +1754,10 @@ err: static void acpi_ec_remove(struct platform_device *pdev) { - struct acpi_device *device = ACPI_COMPANION(&pdev->dev); struct acpi_ec *ec = platform_get_drvdata(pdev); release_region(ec->data_addr, 1); release_region(ec->command_addr, 1); - device->driver_data = NULL; if (ec != boot_ec) { ec_remove_handlers(ec); acpi_ec_free(ec); diff --git a/drivers/acpi/power.c b/drivers/acpi/power.c index 361a7721a6a8..7da5ae5594a7 100644 --- a/drivers/acpi/power.c +++ b/drivers/acpi/power.c @@ -1113,6 +1113,19 @@ static const struct dmi_system_id dmi_leave_unused_power_resources_on[] = { DMI_MATCH(DMI_PRODUCT_NAME, "SATELLITE Click Mini L9W-B"), }, }, + { + /* + * THUNDEROBOT ZERO laptop: Due to its SSDT table bug, power + * resource 'PXP' will be shut down on initialization, making + * the NVMe #2 and the NVIDIA dGPU both unavailable (they're + * both controlled by 'PXP'). + */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "THUNDEROBOT"), + DMI_MATCH(DMI_PRODUCT_NAME, "ZERO"), + } + + }, {} }; diff --git a/drivers/acpi/sbshc.c b/drivers/acpi/sbshc.c index 7fc8ae7396d3..cb60d1d52e3e 100644 --- a/drivers/acpi/sbshc.c +++ b/drivers/acpi/sbshc.c @@ -275,13 +275,11 @@ static int acpi_smbus_hc_probe(struct platform_device *pdev) static void acpi_smbus_hc_remove(struct platform_device *pdev) { - struct acpi_device *device = ACPI_COMPANION(&pdev->dev); struct acpi_smb_hc *hc = platform_get_drvdata(pdev); acpi_ec_remove_query_handler(hc->ec, hc->query_bit); acpi_os_wait_events_complete(); kfree(hc); - device->driver_data = NULL; } module_platform_driver(acpi_smb_hc_driver); diff --git a/drivers/acpi/x86/utils.c b/drivers/acpi/x86/utils.c index 4ee30c2897a2..418951639f51 100644 --- a/drivers/acpi/x86/utils.c +++ b/drivers/acpi/x86/utils.c @@ -81,6 +81,18 @@ static const struct override_status_id override_status_ids[] = { DMI_MATCH(DMI_PRODUCT_NAME, "Mipad2"), }), + /* + * Lenovo Yoga Book uses PWM2 for touch keyboard backlight control. + * It needs to be enabled only for the Android device version (YB1-X90* + * aka YETI-11); the Windows version (YB1-X91*) uses ACPI control + * methods. + */ + PRESENT_ENTRY_HID("80862289", "2", INTEL_ATOM_AIRMONT, { + DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Intel Corporation"), + DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "CHERRYVIEW D1 PLATFORM"), + DMI_EXACT_MATCH(DMI_PRODUCT_VERSION, "YETI-11"), + }), + /* * The INT0002 device is necessary to clear wakeup interrupt sources * on Cherry Trail devices, without it we get nobody cared IRQ msgs. diff --git a/drivers/base/power/wakeirq.c b/drivers/base/power/wakeirq.c index c0809d18fc54..aab843f6dfcc 100644 --- a/drivers/base/power/wakeirq.c +++ b/drivers/base/power/wakeirq.c @@ -273,8 +273,10 @@ EXPORT_SYMBOL_GPL(dev_pm_set_dedicated_wake_irq_reverse); * otherwise try to disable already disabled wakeirq. The wake-up interrupt * starts disabled with IRQ_NOAUTOEN set. * - * Should be only called from rpm_suspend() and rpm_resume() path. - * Caller must hold &dev->power.lock to change wirq->status + * Should be called from rpm_suspend(), rpm_resume(), + * pm_runtime_force_suspend() or pm_runtime_force_resume(). + * Caller must hold &dev->power.lock or disable runtime PM to change + * wirq->status. */ void dev_pm_enable_wake_irq_check(struct device *dev, bool can_change_status) @@ -306,7 +308,8 @@ enable: * @cond_disable: if set, also check WAKE_IRQ_DEDICATED_REVERSE * * Disables wake-up interrupt conditionally based on status. - * Should be only called from rpm_suspend() and rpm_resume() path. + * Should be called from rpm_suspend(), rpm_resume(), + * pm_runtime_force_suspend() or pm_runtime_force_resume(). */ void dev_pm_disable_wake_irq_check(struct device *dev, bool cond_disable) { @@ -332,7 +335,7 @@ void dev_pm_disable_wake_irq_check(struct device *dev, bool cond_disable) * enable wake IRQ after running ->runtime_suspend() which depends on * WAKE_IRQ_DEDICATED_REVERSE. * - * Should be only called from rpm_suspend() path. + * Should be called from rpm_suspend() or pm_runtime_force_suspend(). */ void dev_pm_enable_wake_irq_complete(struct device *dev) { diff --git a/drivers/cpuidle/cpuidle.c b/drivers/cpuidle/cpuidle.c index 18566a496064..c4c4f08b4196 100644 --- a/drivers/cpuidle/cpuidle.c +++ b/drivers/cpuidle/cpuidle.c @@ -372,6 +372,16 @@ noinstr int cpuidle_enter_state(struct cpuidle_device *dev, int cpuidle_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, bool *stop_tick) { + /* + * If there is only a single idle state (or none), there is nothing + * meaningful for the governor to choose. Skip the governor and + * always use state 0 with the tick running. + */ + if (drv->state_count <= 1) { + *stop_tick = false; + return 0; + } + return cpuidle_curr_governor->select(drv, dev, stop_tick); } diff --git a/drivers/cpuidle/governors/haltpoll.c b/drivers/cpuidle/governors/haltpoll.c index 663b7f164d20..b367d10279c8 100644 --- a/drivers/cpuidle/governors/haltpoll.c +++ b/drivers/cpuidle/governors/haltpoll.c @@ -50,9 +50,7 @@ static int haltpoll_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, bool *stop_tick) { - s64 latency_req = cpuidle_governor_latency_req(dev->cpu); - - if (!drv->state_count || latency_req == 0) { + if (cpuidle_governor_latency_req(dev->cpu) == 0) { *stop_tick = false; return 0; } diff --git a/drivers/cpuidle/governors/menu.c b/drivers/cpuidle/governors/menu.c index c6052055ba0f..899ff16ff1fe 100644 --- a/drivers/cpuidle/governors/menu.c +++ b/drivers/cpuidle/governors/menu.c @@ -281,7 +281,7 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, data->bucket = BUCKETS - 1; } - if (drv->state_count <= 1 || latency_req == 0 || + if (latency_req == 0 || ((data->next_timer_ns < drv->states[1].target_residency_ns || latency_req < drv->states[1].exit_latency_ns) && !dev->states_usage[0].disable)) { diff --git a/drivers/cpuidle/governors/teo.c b/drivers/cpuidle/governors/teo.c index 80f3ba942a06..bec0142377b8 100644 --- a/drivers/cpuidle/governors/teo.c +++ b/drivers/cpuidle/governors/teo.c @@ -338,12 +338,6 @@ static int teo_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, */ cpu_data->sleep_length_ns = KTIME_MAX; - /* Check if there is any choice in the first place. */ - if (drv->state_count < 2) { - idx = 0; - goto out_tick; - } - if (!dev->states_usage[0].disable) idx = 0; diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index 3ff6da3bf4e6..539625531709 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -254,7 +254,7 @@ static void rapl_init_domains(struct rapl_package *rp); static int rapl_read_data_raw(struct rapl_domain *rd, enum rapl_primitives prim, bool xlate, u64 *data, - bool atomic); + bool pmu_ctx); static int rapl_write_data_raw(struct rapl_domain *rd, enum rapl_primitives prim, unsigned long long value); @@ -832,7 +832,7 @@ prim_fixups(struct rapl_domain *rd, enum rapl_primitives prim) */ static int rapl_read_data_raw(struct rapl_domain *rd, enum rapl_primitives prim, bool xlate, u64 *data, - bool atomic) + bool pmu_ctx) { u64 value; enum rapl_primitives prim_fixed = prim_fixups(rd, prim); @@ -854,7 +854,7 @@ static int rapl_read_data_raw(struct rapl_domain *rd, ra.mask = rpi->mask; - if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra, atomic)) { + if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra, pmu_ctx)) { pr_debug("failed to read reg 0x%llx for %s:%s\n", ra.reg.val, rd->rp->name, rd->name); return -EIO; } @@ -1590,23 +1590,21 @@ static struct rapl_pmu rapl_pmu; /* PMU helpers */ -static int get_pmu_cpu(struct rapl_package *rp) +static void set_pmu_cpumask(struct rapl_package *rp, cpumask_var_t mask) { int cpu; if (!rp->has_pmu) - return nr_cpu_ids; + return; /* Only TPMI & MSR RAPL are supported for now */ if (rp->priv->type != RAPL_IF_TPMI && rp->priv->type != RAPL_IF_MSR) - return nr_cpu_ids; + return; /* TPMI/MSR RAPL uses any CPU in the package for PMU */ for_each_online_cpu(cpu) if (topology_physical_package_id(cpu) == rp->id) - return cpu; - - return nr_cpu_ids; + cpumask_set_cpu(cpu, mask); } static bool is_rp_pmu_cpu(struct rapl_package *rp, int cpu) @@ -1883,7 +1881,6 @@ static ssize_t cpumask_show(struct device *dev, { struct rapl_package *rp; cpumask_var_t cpu_mask; - int cpu; int ret; if (!alloc_cpumask_var(&cpu_mask, GFP_KERNEL)) @@ -1895,9 +1892,7 @@ static ssize_t cpumask_show(struct device *dev, /* Choose a cpu for each RAPL Package */ list_for_each_entry(rp, &rapl_packages, plist) { - cpu = get_pmu_cpu(rp); - if (cpu < nr_cpu_ids) - cpumask_set_cpu(cpu, cpu_mask); + set_pmu_cpumask(rp, cpu_mask); } cpus_read_unlock(); diff --git a/drivers/powercap/intel_rapl_msr.c b/drivers/powercap/intel_rapl_msr.c index a2bc0a9c1e10..3d5e7f56d68a 100644 --- a/drivers/powercap/intel_rapl_msr.c +++ b/drivers/powercap/intel_rapl_msr.c @@ -110,16 +110,14 @@ static int rapl_cpu_down_prep(unsigned int cpu) return 0; } -static int rapl_msr_read_raw(int cpu, struct reg_action *ra, bool atomic) +static int rapl_msr_read_raw(int cpu, struct reg_action *ra, bool pmu_ctx) { /* - * When called from atomic-context (eg PMU event handler) - * perform MSR read directly using rdmsrq(). + * When called from PMU context, perform MSR read directly using + * rdmsrq() without IPI overhead. Package-scoped MSRs are readable + * from any CPU in the package. */ - if (atomic) { - if (unlikely(smp_processor_id() != cpu)) - return -EIO; - + if (pmu_ctx) { rdmsrq(ra->reg.msr, ra->value); goto out; } diff --git a/drivers/powercap/intel_rapl_tpmi.c b/drivers/powercap/intel_rapl_tpmi.c index 0a0b85f4528b..0f8abdc592bc 100644 --- a/drivers/powercap/intel_rapl_tpmi.c +++ b/drivers/powercap/intel_rapl_tpmi.c @@ -157,7 +157,7 @@ static int parse_one_domain(struct tpmi_rapl_package *trp, u32 offset) tpmi_domain_flags = tpmi_domain_header >> 32 & 0xffff; if (tpmi_domain_version == TPMI_VERSION_INVALID) { - pr_warn(FW_BUG "Invalid version\n"); + pr_debug("Invalid version, other instances may be valid\n"); return -ENODEV; } diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c index bb9398dfa3c1..314fbc1f490f 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c @@ -467,8 +467,11 @@ int proc_thermal_rfim_add(struct pci_dev *pdev, struct proc_thermal_device *proc break; } ret = sysfs_create_group(&pdev->dev.kobj, &dlvr_attribute_group); - if (ret) + if (ret) { + if (proc_priv->mmio_feature_mask & PROC_THERMAL_FEATURE_FIVR) + sysfs_remove_group(&pdev->dev.kobj, &fivr_attribute_group); return ret; + } } if (proc_priv->mmio_feature_mask & PROC_THERMAL_FEATURE_DVFS) { diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c index 63b75d214210..ce09ff3e020f 100644 --- a/fs/ceph/addr.c +++ b/fs/ceph/addr.c @@ -1000,7 +1000,8 @@ unsigned int ceph_define_write_size(struct address_space *mapping) { struct inode *inode = mapping->host; struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); - unsigned int wsize = i_blocksize(inode); + struct ceph_inode_info *ci = ceph_inode(inode); + unsigned int wsize = ci->i_layout.stripe_unit; if (fsc->mount_options->wsize < wsize) wsize = fsc->mount_options->wsize; @@ -1283,16 +1284,16 @@ static inline int move_dirty_folio_in_page_array(struct address_space *mapping, } static -int ceph_process_folio_batch(struct address_space *mapping, - struct writeback_control *wbc, - struct ceph_writeback_ctl *ceph_wbc) +void ceph_process_folio_batch(struct address_space *mapping, + struct writeback_control *wbc, + struct ceph_writeback_ctl *ceph_wbc) { struct inode *inode = mapping->host; struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); struct ceph_client *cl = fsc->client; struct folio *folio = NULL; unsigned i; - int rc = 0; + int rc; for (i = 0; can_next_page_be_processed(ceph_wbc, i); i++) { folio = ceph_wbc->fbatch.folios[i]; @@ -1322,12 +1323,10 @@ int ceph_process_folio_batch(struct address_space *mapping, rc = ceph_check_page_before_write(mapping, wbc, ceph_wbc, folio); if (rc == -ENODATA) { - rc = 0; folio_unlock(folio); ceph_wbc->fbatch.folios[i] = NULL; continue; } else if (rc == -E2BIG) { - rc = 0; folio_unlock(folio); ceph_wbc->fbatch.folios[i] = NULL; break; @@ -1379,8 +1378,6 @@ int ceph_process_folio_batch(struct address_space *mapping, } ceph_wbc->processed_in_fbatch = i; - - return rc; } static inline @@ -1666,7 +1663,9 @@ retry: tag_pages_for_writeback(mapping, ceph_wbc.index, ceph_wbc.end); while (!has_writeback_done(&ceph_wbc)) { - ceph_wbc.locked_pages = 0; + BUG_ON(ceph_wbc.locked_pages); + BUG_ON(ceph_wbc.pages); + ceph_wbc.max_pages = ceph_wbc.wsize >> PAGE_SHIFT; get_more_pages: @@ -1684,10 +1683,8 @@ get_more_pages: break; process_folio_batch: - rc = ceph_process_folio_batch(mapping, wbc, &ceph_wbc); + ceph_process_folio_batch(mapping, wbc, &ceph_wbc); ceph_shift_unused_folios_left(&ceph_wbc.fbatch); - if (rc) - goto release_folios; /* did we get anything? */ if (!ceph_wbc.locked_pages) @@ -2199,6 +2196,7 @@ int ceph_uninline_data(struct file *file) struct ceph_osd_request *req = NULL; struct ceph_cap_flush *prealloc_cf = NULL; struct folio *folio = NULL; + struct ceph_snap_context *snapc = NULL; u64 inline_version = CEPH_INLINE_NONE; struct page *pages[1]; int err = 0; @@ -2226,6 +2224,24 @@ int ceph_uninline_data(struct file *file) if (inline_version == 1) /* initial version, no data */ goto out_uninline; + down_read(&fsc->mdsc->snap_rwsem); + spin_lock(&ci->i_ceph_lock); + if (__ceph_have_pending_cap_snap(ci)) { + struct ceph_cap_snap *capsnap = + list_last_entry(&ci->i_cap_snaps, + struct ceph_cap_snap, + ci_item); + snapc = ceph_get_snap_context(capsnap->context); + } else { + if (!ci->i_head_snapc) { + ci->i_head_snapc = ceph_get_snap_context( + ci->i_snap_realm->cached_context); + } + snapc = ceph_get_snap_context(ci->i_head_snapc); + } + spin_unlock(&ci->i_ceph_lock); + up_read(&fsc->mdsc->snap_rwsem); + folio = read_mapping_folio(inode->i_mapping, 0, file); if (IS_ERR(folio)) { err = PTR_ERR(folio); @@ -2241,7 +2257,7 @@ int ceph_uninline_data(struct file *file) req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, ceph_vino(inode), 0, &len, 0, 1, CEPH_OSD_OP_CREATE, CEPH_OSD_FLAG_WRITE, - NULL, 0, 0, false); + snapc, 0, 0, false); if (IS_ERR(req)) { err = PTR_ERR(req); goto out_unlock; @@ -2257,7 +2273,7 @@ int ceph_uninline_data(struct file *file) req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, ceph_vino(inode), 0, &len, 1, 3, CEPH_OSD_OP_WRITE, CEPH_OSD_FLAG_WRITE, - NULL, ci->i_truncate_seq, + snapc, ci->i_truncate_seq, ci->i_truncate_size, false); if (IS_ERR(req)) { err = PTR_ERR(req); @@ -2320,6 +2336,7 @@ out_unlock: folio_put(folio); } out: + ceph_put_snap_context(snapc); ceph_free_cap_flush(prealloc_cf); doutc(cl, "%llx.%llx inline_version %llu = %d\n", ceph_vinop(inode), inline_version, err); diff --git a/fs/ceph/file.c b/fs/ceph/file.c index 31b691b2aea2..72cc67ab4ead 100644 --- a/fs/ceph/file.c +++ b/fs/ceph/file.c @@ -2568,6 +2568,7 @@ static int ceph_zero_partial_object(struct inode *inode, struct ceph_inode_info *ci = ceph_inode(inode); struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); struct ceph_osd_request *req; + struct ceph_snap_context *snapc; int ret = 0; loff_t zero = 0; int op; @@ -2582,12 +2583,25 @@ static int ceph_zero_partial_object(struct inode *inode, op = CEPH_OSD_OP_ZERO; } + spin_lock(&ci->i_ceph_lock); + if (__ceph_have_pending_cap_snap(ci)) { + struct ceph_cap_snap *capsnap = + list_last_entry(&ci->i_cap_snaps, + struct ceph_cap_snap, + ci_item); + snapc = ceph_get_snap_context(capsnap->context); + } else { + BUG_ON(!ci->i_head_snapc); + snapc = ceph_get_snap_context(ci->i_head_snapc); + } + spin_unlock(&ci->i_ceph_lock); + req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, ceph_vino(inode), offset, length, 0, 1, op, CEPH_OSD_FLAG_WRITE, - NULL, 0, 0, false); + snapc, 0, 0, false); if (IS_ERR(req)) { ret = PTR_ERR(req); goto out; @@ -2601,6 +2615,7 @@ static int ceph_zero_partial_object(struct inode *inode, ceph_osdc_put_request(req); out: + ceph_put_snap_context(snapc); return ret; } diff --git a/fs/ntfs3/attrib.c b/fs/ntfs3/attrib.c index 980ae9157248..6cb9bc5d605c 100644 --- a/fs/ntfs3/attrib.c +++ b/fs/ntfs3/attrib.c @@ -91,7 +91,8 @@ static int attr_load_runs(struct ATTRIB *attr, struct ntfs_inode *ni, * run_deallocate_ex - Deallocate clusters. */ static int run_deallocate_ex(struct ntfs_sb_info *sbi, struct runs_tree *run, - CLST vcn, CLST len, CLST *done, bool trim) + CLST vcn, CLST len, CLST *done, bool trim, + struct runs_tree *run_da) { int err = 0; CLST vcn_next, vcn0 = vcn, lcn, clen, dn = 0; @@ -120,6 +121,16 @@ failed: if (sbi) { /* mark bitmap range [lcn + clen) as free and trim clusters. */ mark_as_free_ex(sbi, lcn, clen, trim); + + if (run_da) { + CLST da_len; + if (!run_remove_range(run_da, vcn, clen, + &da_len)) { + err = -ENOMEM; + goto failed; + } + ntfs_sub_da(sbi, da_len); + } } dn += clen; } @@ -147,9 +158,10 @@ out: * attr_allocate_clusters - Find free space, mark it as used and store in @run. */ int attr_allocate_clusters(struct ntfs_sb_info *sbi, struct runs_tree *run, - CLST vcn, CLST lcn, CLST len, CLST *pre_alloc, - enum ALLOCATE_OPT opt, CLST *alen, const size_t fr, - CLST *new_lcn, CLST *new_len) + struct runs_tree *run_da, CLST vcn, CLST lcn, + CLST len, CLST *pre_alloc, enum ALLOCATE_OPT opt, + CLST *alen, const size_t fr, CLST *new_lcn, + CLST *new_len) { int err; CLST flen, vcn0 = vcn, pre = pre_alloc ? *pre_alloc : 0; @@ -166,6 +178,12 @@ int attr_allocate_clusters(struct ntfs_sb_info *sbi, struct runs_tree *run, continue; } + if (err == -ENOSPC && new_len && vcn - vcn0) { + /* Keep already allocated clusters. */ + *alen = vcn - vcn0; + return 0; + } + if (err) goto out; @@ -179,12 +197,21 @@ int attr_allocate_clusters(struct ntfs_sb_info *sbi, struct runs_tree *run, /* Add new fragment into run storage. */ if (!run_add_entry(run, vcn, lcn, flen, opt & ALLOCATE_MFT)) { +undo_alloc: /* Undo last 'ntfs_look_for_free_space' */ mark_as_free_ex(sbi, lcn, len, false); err = -ENOMEM; goto out; } + if (run_da) { + CLST da_len; + if (!run_remove_range(run_da, vcn, flen, &da_len)) { + goto undo_alloc; + } + ntfs_sub_da(sbi, da_len); + } + if (opt & ALLOCATE_ZERO) { u8 shift = sbi->cluster_bits - SECTOR_SHIFT; @@ -199,7 +226,7 @@ int attr_allocate_clusters(struct ntfs_sb_info *sbi, struct runs_tree *run, vcn += flen; if (flen >= len || (opt & ALLOCATE_MFT) || - (fr && run->count - cnt >= fr)) { + (opt & ALLOCATE_ONE_FR) || (fr && run->count - cnt >= fr)) { *alen = vcn - vcn0; return 0; } @@ -210,7 +237,8 @@ int attr_allocate_clusters(struct ntfs_sb_info *sbi, struct runs_tree *run, out: /* Undo 'ntfs_look_for_free_space' */ if (vcn - vcn0) { - run_deallocate_ex(sbi, run, vcn0, vcn - vcn0, NULL, false); + run_deallocate_ex(sbi, run, vcn0, vcn - vcn0, NULL, false, + run_da); run_truncate(run, vcn0); } @@ -275,7 +303,7 @@ int attr_make_nonresident(struct ntfs_inode *ni, struct ATTRIB *attr, } else { const char *data = resident_data(attr); - err = attr_allocate_clusters(sbi, run, 0, 0, len, NULL, + err = attr_allocate_clusters(sbi, run, NULL, 0, 0, len, NULL, ALLOCATE_DEF, &alen, 0, NULL, NULL); if (err) @@ -391,7 +419,7 @@ static int attr_set_size_res(struct ntfs_inode *ni, struct ATTRIB *attr, } /* - * attr_set_size - Change the size of attribute. + * attr_set_size_ex - Change the size of attribute. * * Extend: * - Sparse/compressed: No allocated clusters. @@ -399,24 +427,28 @@ static int attr_set_size_res(struct ntfs_inode *ni, struct ATTRIB *attr, * Shrink: * - No deallocate if @keep_prealloc is set. */ -int attr_set_size(struct ntfs_inode *ni, enum ATTR_TYPE type, - const __le16 *name, u8 name_len, struct runs_tree *run, - u64 new_size, const u64 *new_valid, bool keep_prealloc, - struct ATTRIB **ret) +int attr_set_size_ex(struct ntfs_inode *ni, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, struct runs_tree *run, + u64 new_size, const u64 *new_valid, bool keep_prealloc, + struct ATTRIB **ret, bool no_da) { int err = 0; struct ntfs_sb_info *sbi = ni->mi.sbi; u8 cluster_bits = sbi->cluster_bits; bool is_mft = ni->mi.rno == MFT_REC_MFT && type == ATTR_DATA && !name_len; - u64 old_valid, old_size, old_alloc, new_alloc, new_alloc_tmp; + u64 old_valid, old_size, old_alloc, new_alloc_tmp; + u64 new_alloc = 0; struct ATTRIB *attr = NULL, *attr_b; struct ATTR_LIST_ENTRY *le, *le_b; struct mft_inode *mi, *mi_b; CLST alen, vcn, lcn, new_alen, old_alen, svcn, evcn; CLST next_svcn, pre_alloc = -1, done = 0; - bool is_ext, is_bad = false; + bool is_ext = false, is_bad = false; bool dirty = false; + struct runs_tree *run_da = run == &ni->file.run ? &ni->file.run_da : + NULL; + bool da = !is_mft && sbi->options->delalloc && run_da && !no_da; u32 align; struct MFT_REC *rec; @@ -448,8 +480,11 @@ again: is_ext = is_attr_ext(attr_b); align = sbi->cluster_size; - if (is_ext) + if (is_ext) { align <<= attr_b->nres.c_unit; + keep_prealloc = false; + da = false; + } old_valid = le64_to_cpu(attr_b->nres.valid_size); old_size = le64_to_cpu(attr_b->nres.data_size); @@ -467,6 +502,37 @@ again_1: goto ok; } + if (da && + (vcn = old_alen + run_len(&ni->file.run_da), new_alen > vcn)) { + /* Resize up normal file. Delay new clusters allocation. */ + alen = new_alen - vcn; + + if (ntfs_check_free_space(sbi, alen, 0, true)) { + if (!run_add_entry(&ni->file.run_da, vcn, SPARSE_LCN, + alen, false)) { + err = -ENOMEM; + goto out; + } + + ntfs_add_da(sbi, alen); + goto ok1; + } + } + + if (!keep_prealloc && run_da && run_da->count && + (vcn = run_get_max_vcn(run_da), new_alen < vcn)) { + /* Shrink delayed clusters. */ + + /* Try to remove fragment from delay allocated run. */ + if (!run_remove_range(run_da, new_alen, vcn - new_alen, + &alen)) { + err = -ENOMEM; + goto out; + } + + ntfs_sub_da(sbi, alen); + } + vcn = old_alen - 1; svcn = le64_to_cpu(attr_b->nres.svcn); @@ -572,7 +638,8 @@ add_alloc_in_same_attr_seg: } else { /* ~3 bytes per fragment. */ err = attr_allocate_clusters( - sbi, run, vcn, lcn, to_allocate, &pre_alloc, + sbi, run, run_da, vcn, lcn, to_allocate, + &pre_alloc, is_mft ? ALLOCATE_MFT : ALLOCATE_DEF, &alen, is_mft ? 0 : (sbi->record_size - @@ -751,14 +818,14 @@ pack_runs: mi_b->dirty = dirty = true; err = run_deallocate_ex(sbi, run, vcn, evcn - vcn + 1, &dlen, - true); + true, run_da); if (err) goto out; if (is_ext) { /* dlen - really deallocated clusters. */ le64_sub_cpu(&attr_b->nres.total_size, - ((u64)dlen << cluster_bits)); + (u64)dlen << cluster_bits); } run_truncate(run, vcn); @@ -813,14 +880,14 @@ ok1: if (((type == ATTR_DATA && !name_len) || (type == ATTR_ALLOC && name == I30_NAME))) { /* Update inode_set_bytes. */ - if (attr_b->non_res) { - new_alloc = le64_to_cpu(attr_b->nres.alloc_size); - if (inode_get_bytes(&ni->vfs_inode) != new_alloc) { - inode_set_bytes(&ni->vfs_inode, new_alloc); - dirty = true; - } + if (attr_b->non_res && + inode_get_bytes(&ni->vfs_inode) != new_alloc) { + inode_set_bytes(&ni->vfs_inode, new_alloc); + dirty = true; } + i_size_write(&ni->vfs_inode, new_size); + /* Don't forget to update duplicate information in parent. */ if (dirty) { ni->ni_flags |= NI_FLAG_UPDATE_PARENT; @@ -861,7 +928,7 @@ restore_run: is_bad = true; undo_1: - run_deallocate_ex(sbi, run, vcn, alen, NULL, false); + run_deallocate_ex(sbi, run, vcn, alen, NULL, false, run_da); run_truncate(run, vcn); out: @@ -884,43 +951,74 @@ bad_inode: * - new allocated clusters are zeroed via blkdev_issue_zeroout. */ int attr_data_get_block(struct ntfs_inode *ni, CLST vcn, CLST clen, CLST *lcn, - CLST *len, bool *new, bool zero) + CLST *len, bool *new, bool zero, void **res, bool no_da) { - int err = 0; - struct runs_tree *run = &ni->file.run; - struct ntfs_sb_info *sbi; - u8 cluster_bits; - struct ATTRIB *attr, *attr_b; - struct ATTR_LIST_ENTRY *le, *le_b; - struct mft_inode *mi, *mi_b; - CLST hint, svcn, to_alloc, evcn1, next_svcn, asize, end, vcn0, alen; - CLST alloc, evcn; - unsigned fr; - u64 total_size, total_size0; - int step = 0; + int err; if (new) *new = false; + if (res) + *res = NULL; /* Try to find in cache. */ down_read(&ni->file.run_lock); - if (!run_lookup_entry(run, vcn, lcn, len, NULL)) + if (!no_da && run_lookup_entry(&ni->file.run_da, vcn, lcn, len, NULL)) { + /* The requested vcn is delay allocated. */ + *lcn = DELALLOC_LCN; + } else if (run_lookup_entry(&ni->file.run, vcn, lcn, len, NULL)) { + /* The requested vcn is known in current run. */ + } else { *len = 0; + } up_read(&ni->file.run_lock); if (*len && (*lcn != SPARSE_LCN || !new)) return 0; /* Fast normal way without allocation. */ /* No cluster in cache or we need to allocate cluster in hole. */ - sbi = ni->mi.sbi; - cluster_bits = sbi->cluster_bits; - ni_lock(ni); down_write(&ni->file.run_lock); - /* Repeat the code above (under write lock). */ - if (!run_lookup_entry(run, vcn, lcn, len, NULL)) + err = attr_data_get_block_locked(ni, vcn, clen, lcn, len, new, zero, + res, no_da); + + up_write(&ni->file.run_lock); + ni_unlock(ni); + + return err; +} + +/* + * attr_data_get_block_locked - Helper for attr_data_get_block. + */ +int attr_data_get_block_locked(struct ntfs_inode *ni, CLST vcn, CLST clen, + CLST *lcn, CLST *len, bool *new, bool zero, + void **res, bool no_da) +{ + int err = 0; + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct runs_tree *run = &ni->file.run; + struct runs_tree *run_da = &ni->file.run_da; + bool da = sbi->options->delalloc && !no_da; + u8 cluster_bits; + struct ATTRIB *attr, *attr_b; + struct ATTR_LIST_ENTRY *le, *le_b; + struct mft_inode *mi, *mi_b; + CLST hint, svcn, to_alloc, evcn1, next_svcn, asize, end, vcn0; + CLST alloc, evcn; + unsigned fr; + u64 total_size, total_size0; + int step; + +again: + if (da && run_lookup_entry(run_da, vcn, lcn, len, NULL)) { + /* The requested vcn is delay allocated. */ + *lcn = DELALLOC_LCN; + } else if (run_lookup_entry(run, vcn, lcn, len, NULL)) { + /* The requested vcn is known in current run. */ + } else { *len = 0; + } if (*len) { if (*lcn != SPARSE_LCN || !new) @@ -929,6 +1027,9 @@ int attr_data_get_block(struct ntfs_inode *ni, CLST vcn, CLST clen, CLST *lcn, clen = *len; } + cluster_bits = sbi->cluster_bits; + step = 0; + le_b = NULL; attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, NULL, 0, NULL, &mi_b); if (!attr_b) { @@ -937,8 +1038,15 @@ int attr_data_get_block(struct ntfs_inode *ni, CLST vcn, CLST clen, CLST *lcn, } if (!attr_b->non_res) { + u32 data_size = le32_to_cpu(attr_b->res.data_size); *lcn = RESIDENT_LCN; - *len = 1; + *len = data_size; + if (res && data_size) { + *res = kmemdup(resident_data(attr_b), data_size, + GFP_KERNEL); + if (!*res) + err = -ENOMEM; + } goto out; } @@ -948,7 +1056,7 @@ int attr_data_get_block(struct ntfs_inode *ni, CLST vcn, CLST clen, CLST *lcn, err = -EINVAL; } else { *len = 1; - *lcn = SPARSE_LCN; + *lcn = EOF_LCN; } goto out; } @@ -1026,7 +1134,8 @@ int attr_data_get_block(struct ntfs_inode *ni, CLST vcn, CLST clen, CLST *lcn, to_alloc = ((vcn0 + clen + clst_per_frame - 1) & cmask) - vcn; if (fr < clst_per_frame) fr = clst_per_frame; - zero = true; + if (vcn != vcn0) + zero = true; /* Check if 'vcn' and 'vcn0' in different attribute segments. */ if (vcn < svcn || evcn1 <= vcn) { @@ -1043,11 +1152,38 @@ int attr_data_get_block(struct ntfs_inode *ni, CLST vcn, CLST clen, CLST *lcn, if (err) goto out; } + da = false; /* no delalloc for compressed file. */ } if (vcn + to_alloc > asize) to_alloc = asize - vcn; + if (da) { + CLST rlen1, rlen2; + if (!ntfs_check_free_space(sbi, to_alloc, 0, true)) { + err = ni_allocate_da_blocks_locked(ni); + if (err) + goto out; + /* Layout of records may be changed. Start again without 'da'. */ + da = false; + goto again; + } + + /* run_add_entry consolidates existed ranges. */ + rlen1 = run_len(run_da); + if (!run_add_entry(run_da, vcn, SPARSE_LCN, to_alloc, false)) { + err = -ENOMEM; + goto out; + } + rlen2 = run_len(run_da); + + /* new added delay clusters = rlen2 - rlen1. */ + ntfs_add_da(sbi, rlen2 - rlen1); + *len = to_alloc; + *lcn = DELALLOC_LCN; + goto ok; + } + /* Get the last LCN to allocate from. */ hint = 0; @@ -1062,18 +1198,19 @@ int attr_data_get_block(struct ntfs_inode *ni, CLST vcn, CLST clen, CLST *lcn, } /* Allocate and zeroout new clusters. */ - err = attr_allocate_clusters(sbi, run, vcn, hint + 1, to_alloc, NULL, - zero ? ALLOCATE_ZERO : ALLOCATE_DEF, &alen, - fr, lcn, len); + err = attr_allocate_clusters(sbi, run, run_da, vcn, hint + 1, to_alloc, + NULL, + zero ? ALLOCATE_ZERO : ALLOCATE_ONE_FR, + len, fr, lcn, len); if (err) goto out; *new = true; step = 1; - end = vcn + alen; + end = vcn + *len; /* Save 'total_size0' to restore if error. */ total_size0 = le64_to_cpu(attr_b->nres.total_size); - total_size = total_size0 + ((u64)alen << cluster_bits); + total_size = total_size0 + ((u64)*len << cluster_bits); if (vcn != vcn0) { if (!run_lookup_entry(run, vcn0, lcn, len, NULL)) { @@ -1139,7 +1276,7 @@ repack: * in 'ni_insert_nonresident'. * Return in advance -ENOSPC here if there are no free cluster and no free MFT. */ - if (!ntfs_check_for_free_space(sbi, 1, 1)) { + if (!ntfs_check_free_space(sbi, 1, 1, false)) { /* Undo step 1. */ err = -ENOSPC; goto undo1; @@ -1224,8 +1361,6 @@ out: /* Too complex to restore. */ _ntfs_bad_inode(&ni->vfs_inode); } - up_write(&ni->file.run_lock); - ni_unlock(ni); return err; @@ -1234,41 +1369,14 @@ undo1: attr_b->nres.total_size = cpu_to_le64(total_size0); inode_set_bytes(&ni->vfs_inode, total_size0); - if (run_deallocate_ex(sbi, run, vcn, alen, NULL, false) || - !run_add_entry(run, vcn, SPARSE_LCN, alen, false) || + if (run_deallocate_ex(sbi, run, vcn, *len, NULL, false, run_da) || + !run_add_entry(run, vcn, SPARSE_LCN, *len, false) || mi_pack_runs(mi, attr, run, max(end, evcn1) - svcn)) { _ntfs_bad_inode(&ni->vfs_inode); } goto out; } -int attr_data_read_resident(struct ntfs_inode *ni, struct folio *folio) -{ - u64 vbo; - struct ATTRIB *attr; - u32 data_size; - size_t len; - - attr = ni_find_attr(ni, NULL, NULL, ATTR_DATA, NULL, 0, NULL, NULL); - if (!attr) - return -EINVAL; - - if (attr->non_res) - return E_NTFS_NONRESIDENT; - - vbo = folio->index << PAGE_SHIFT; - data_size = le32_to_cpu(attr->res.data_size); - if (vbo > data_size) - len = 0; - else - len = min(data_size - vbo, folio_size(folio)); - - folio_fill_tail(folio, 0, resident_data(attr) + vbo, len); - folio_mark_uptodate(folio); - - return 0; -} - int attr_data_write_resident(struct ntfs_inode *ni, struct folio *folio) { u64 vbo; @@ -1285,7 +1393,7 @@ int attr_data_write_resident(struct ntfs_inode *ni, struct folio *folio) return E_NTFS_NONRESIDENT; } - vbo = folio->index << PAGE_SHIFT; + vbo = folio_pos(folio); data_size = le32_to_cpu(attr->res.data_size); if (vbo < data_size) { char *data = resident_data(attr); @@ -1354,19 +1462,27 @@ int attr_load_runs_range(struct ntfs_inode *ni, enum ATTR_TYPE type, CLST vcn; CLST vcn_last = (to - 1) >> cluster_bits; CLST lcn, clen; - int err; + int err = 0; + int retry = 0; for (vcn = from >> cluster_bits; vcn <= vcn_last; vcn += clen) { - if (!run_lookup_entry(run, vcn, &lcn, &clen, NULL)) { - err = attr_load_runs_vcn(ni, type, name, name_len, run, - vcn); - if (err) - return err; - clen = 0; /* Next run_lookup_entry(vcn) must be success. */ + if (run_lookup_entry(run, vcn, &lcn, &clen, NULL)) { + retry = 0; + continue; } + if (retry) { + err = -EINVAL; + break; + } + err = attr_load_runs_vcn(ni, type, name, name_len, run, vcn); + if (err) + break; + + clen = 0; /* Next run_lookup_entry(vcn) must be success. */ + retry++; } - return 0; + return err; } #ifdef CONFIG_NTFS3_LZX_XPRESS @@ -1689,7 +1805,7 @@ int attr_allocate_frame(struct ntfs_inode *ni, CLST frame, size_t compr_size, if (len < clst_data) { err = run_deallocate_ex(sbi, run, vcn + len, clst_data - len, - NULL, true); + NULL, true, NULL); if (err) goto out; @@ -1709,7 +1825,7 @@ int attr_allocate_frame(struct ntfs_inode *ni, CLST frame, size_t compr_size, hint = -1; } - err = attr_allocate_clusters(sbi, run, vcn + clst_data, + err = attr_allocate_clusters(sbi, run, NULL, vcn + clst_data, hint + 1, len - clst_data, NULL, ALLOCATE_DEF, &alen, 0, NULL, NULL); @@ -1864,6 +1980,7 @@ int attr_collapse_range(struct ntfs_inode *ni, u64 vbo, u64 bytes) CLST vcn, end; u64 valid_size, data_size, alloc_size, total_size; u32 mask; + u64 i_size; __le16 a_flags; if (!bytes) @@ -1879,52 +1996,79 @@ int attr_collapse_range(struct ntfs_inode *ni, u64 vbo, u64 bytes) return 0; } - data_size = le64_to_cpu(attr_b->nres.data_size); - alloc_size = le64_to_cpu(attr_b->nres.alloc_size); - a_flags = attr_b->flags; - - if (is_attr_ext(attr_b)) { - total_size = le64_to_cpu(attr_b->nres.total_size); - mask = (sbi->cluster_size << attr_b->nres.c_unit) - 1; - } else { - total_size = alloc_size; - mask = sbi->cluster_mask; - } - - if ((vbo & mask) || (bytes & mask)) { + mask = is_attr_ext(attr_b) ? + ((sbi->cluster_size << attr_b->nres.c_unit) - 1) : + sbi->cluster_mask; + if ((vbo | bytes) & mask) { /* Allow to collapse only cluster aligned ranges. */ return -EINVAL; } - if (vbo > data_size) + /* i_size - size of file with delay allocated clusters. */ + i_size = ni->vfs_inode.i_size; + + if (vbo > i_size) return -EINVAL; down_write(&ni->file.run_lock); - if (vbo + bytes >= data_size) { - u64 new_valid = min(ni->i_valid, vbo); + if (vbo + bytes >= i_size) { + valid_size = min(ni->i_valid, vbo); /* Simple truncate file at 'vbo'. */ truncate_setsize(&ni->vfs_inode, vbo); err = attr_set_size(ni, ATTR_DATA, NULL, 0, &ni->file.run, vbo, - &new_valid, true, NULL); + &valid_size, true); - if (!err && new_valid < ni->i_valid) - ni->i_valid = new_valid; + if (!err && valid_size < ni->i_valid) + ni->i_valid = valid_size; goto out; } - /* - * Enumerate all attribute segments and collapse. - */ - alen = alloc_size >> sbi->cluster_bits; vcn = vbo >> sbi->cluster_bits; len = bytes >> sbi->cluster_bits; end = vcn + len; dealloc = 0; done = 0; + /* + * Check delayed clusters. + */ + if (ni->file.run_da.count) { + struct runs_tree *run_da = &ni->file.run_da; + if (run_is_mapped_full(run_da, vcn, end - 1)) { + /* + * The requested range is full in delayed clusters. + */ + err = attr_set_size_ex(ni, ATTR_DATA, NULL, 0, run, + i_size - bytes, NULL, false, + NULL, true); + goto out; + } + + /* Collapse request crosses real and delayed clusters. */ + err = ni_allocate_da_blocks_locked(ni); + if (err) + goto out; + + /* Layout of records maybe changed. */ + le_b = NULL; + attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, NULL, 0, NULL, + &mi_b); + if (!attr_b || !attr_b->non_res) { + err = -ENOENT; + goto out; + } + } + + data_size = le64_to_cpu(attr_b->nres.data_size); + alloc_size = le64_to_cpu(attr_b->nres.alloc_size); + total_size = is_attr_ext(attr_b) ? + le64_to_cpu(attr_b->nres.total_size) : + alloc_size; + alen = alloc_size >> sbi->cluster_bits; + a_flags = attr_b->flags; svcn = le64_to_cpu(attr_b->nres.svcn); evcn1 = le64_to_cpu(attr_b->nres.evcn) + 1; @@ -1947,6 +2091,9 @@ int attr_collapse_range(struct ntfs_inode *ni, u64 vbo, u64 bytes) goto out; } + /* + * Enumerate all attribute segments and collapse. + */ for (;;) { CLST vcn1, eat, next_svcn; @@ -1974,13 +2121,13 @@ check_seg: vcn1 = vcn + done; /* original vcn in attr/run. */ eat = min(end, evcn1) - vcn1; - err = run_deallocate_ex(sbi, run, vcn1, eat, &dealloc, true); + err = run_deallocate_ex(sbi, run, vcn1, eat, &dealloc, true, + NULL); if (err) goto out; if (svcn + eat < evcn1) { /* Collapse a part of this attribute segment. */ - if (!run_collapse_range(run, vcn1, eat, done)) { err = -ENOMEM; goto out; @@ -2161,9 +2308,9 @@ int attr_punch_hole(struct ntfs_inode *ni, u64 vbo, u64 bytes, u32 *frame_size) bytes = alloc_size; bytes -= vbo; - if ((vbo & mask) || (bytes & mask)) { + if ((vbo | bytes) & mask) { /* We have to zero a range(s). */ - if (frame_size == NULL) { + if (!frame_size) { /* Caller insists range is aligned. */ return -EINVAL; } @@ -2222,7 +2369,8 @@ int attr_punch_hole(struct ntfs_inode *ni, u64 vbo, u64 bytes, u32 *frame_size) * Calculate how many clusters there are. * Don't do any destructive actions. */ - err = run_deallocate_ex(NULL, run, vcn1, zero, &hole2, false); + err = run_deallocate_ex(NULL, run, vcn1, zero, &hole2, false, + NULL); if (err) goto done; @@ -2260,7 +2408,8 @@ int attr_punch_hole(struct ntfs_inode *ni, u64 vbo, u64 bytes, u32 *frame_size) } /* Real deallocate. Should not fail. */ - run_deallocate_ex(sbi, &run2, vcn1, zero, &hole, true); + run_deallocate_ex(sbi, &run2, vcn1, zero, &hole, true, + &ni->file.run_da); next_attr: /* Free all allocated memory. */ @@ -2372,7 +2521,7 @@ int attr_insert_range(struct ntfs_inode *ni, u64 vbo, u64 bytes) return -EINVAL; } - if ((vbo & mask) || (bytes & mask)) { + if ((vbo | bytes) & mask) { /* Allow to insert only frame aligned ranges. */ return -EINVAL; } @@ -2391,7 +2540,7 @@ int attr_insert_range(struct ntfs_inode *ni, u64 vbo, u64 bytes) if (!attr_b->non_res) { err = attr_set_size(ni, ATTR_DATA, NULL, 0, run, - data_size + bytes, NULL, false, NULL); + data_size + bytes, NULL, false); le_b = NULL; attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, NULL, 0, NULL, @@ -2414,7 +2563,7 @@ int attr_insert_range(struct ntfs_inode *ni, u64 vbo, u64 bytes) goto done; } - /* Resident files becomes nonresident. */ + /* Resident file becomes nonresident. */ data_size = le64_to_cpu(attr_b->nres.data_size); alloc_size = le64_to_cpu(attr_b->nres.alloc_size); } @@ -2451,10 +2600,13 @@ int attr_insert_range(struct ntfs_inode *ni, u64 vbo, u64 bytes) if (err) goto out; - if (!run_insert_range(run, vcn, len)) { - err = -ENOMEM; + err = run_insert_range(run, vcn, len); + if (err) + goto out; + + err = run_insert_range_da(&ni->file.run_da, vcn, len); + if (err) goto out; - } /* Try to pack in current record as much as possible. */ err = mi_pack_runs(mi, attr, run, evcn1 + len - svcn); diff --git a/fs/ntfs3/attrlist.c b/fs/ntfs3/attrlist.c index a4d74bed74fa..270a29323530 100644 --- a/fs/ntfs3/attrlist.c +++ b/fs/ntfs3/attrlist.c @@ -52,6 +52,11 @@ int ntfs_load_attr_list(struct ntfs_inode *ni, struct ATTRIB *attr) if (!attr->non_res) { lsize = le32_to_cpu(attr->res.data_size); + if (!lsize) { + err = -EINVAL; + goto out; + } + /* attr is resident: lsize < record_size (1K or 4K) */ le = kvmalloc(al_aligned(lsize), GFP_KERNEL); if (!le) { @@ -66,6 +71,10 @@ int ntfs_load_attr_list(struct ntfs_inode *ni, struct ATTRIB *attr) u16 run_off = le16_to_cpu(attr->nres.run_off); lsize = le64_to_cpu(attr->nres.data_size); + if (!lsize) { + err = -EINVAL; + goto out; + } run_init(&ni->attr_list.run); @@ -336,8 +345,8 @@ int al_add_le(struct ntfs_inode *ni, enum ATTR_TYPE type, const __le16 *name, le->id = id; memcpy(le->name, name, sizeof(short) * name_len); - err = attr_set_size(ni, ATTR_LIST, NULL, 0, &al->run, new_size, - &new_size, true, &attr); + err = attr_set_size_ex(ni, ATTR_LIST, NULL, 0, &al->run, new_size, + &new_size, true, &attr, false); if (err) { /* Undo memmove above. */ memmove(le, Add2Ptr(le, sz), old_size - off); @@ -395,8 +404,8 @@ int al_update(struct ntfs_inode *ni, int sync) * Attribute list increased on demand in al_add_le. * Attribute list decreased here. */ - err = attr_set_size(ni, ATTR_LIST, NULL, 0, &al->run, al->size, NULL, - false, &attr); + err = attr_set_size_ex(ni, ATTR_LIST, NULL, 0, &al->run, al->size, NULL, + false, &attr, false); if (err) goto out; diff --git a/fs/ntfs3/bitmap.c b/fs/ntfs3/bitmap.c index 65d05e6a0566..db7d0ecfb469 100644 --- a/fs/ntfs3/bitmap.c +++ b/fs/ntfs3/bitmap.c @@ -508,6 +508,8 @@ static int wnd_rescan(struct wnd_bitmap *wnd) size_t wpos, wbit, iw, vbo; struct buffer_head *bh = NULL; CLST lcn, clen; + struct file_ra_state *ra; + struct address_space *mapping = sb->s_bdev->bd_mapping; wnd->uptodated = 0; wnd->extent_max = 0; @@ -516,6 +518,13 @@ static int wnd_rescan(struct wnd_bitmap *wnd) vbo = 0; + /* Allocate in memory instead of stack. Not critical if failed. */ + ra = kzalloc(sizeof(*ra), GFP_NOFS); + if (ra) { + file_ra_state_init(ra, mapping); + ra->ra_pages = (wnd->nbits / 8 + PAGE_SIZE - 1) >> PAGE_SHIFT; + } + for (iw = 0; iw < wnd->nwnd; iw++) { if (iw + 1 == wnd->nwnd) wbits = wnd->bits_last; @@ -552,6 +561,13 @@ static int wnd_rescan(struct wnd_bitmap *wnd) len = ((u64)clen << cluster_bits) - off; } + if (ra) { + pgoff_t idx = lbo >> PAGE_SHIFT; + if (!ra_has_index(ra, idx)) + page_cache_sync_readahead(mapping, ra, NULL, + idx, 1); + } + bh = ntfs_bread(sb, lbo >> sb->s_blocksize_bits); if (!bh) { err = -EIO; @@ -638,6 +654,7 @@ next_wnd: } out: + kfree(ra); return err; } diff --git a/fs/ntfs3/dir.c b/fs/ntfs3/dir.c index 596f8c62f033..4652a56ad105 100644 --- a/fs/ntfs3/dir.c +++ b/fs/ntfs3/dir.c @@ -393,33 +393,77 @@ static int ntfs_read_hdr(struct ntfs_sb_info *sbi, struct ntfs_inode *ni, * ntfs_readdir - file_operations::iterate_shared * * Use non sorted enumeration. - * We have an example of broken volume where sorted enumeration - * counts each name twice. + * Sorted enumeration may result infinite loop if names tree contains loop. */ static int ntfs_readdir(struct file *file, struct dir_context *ctx) { const struct INDEX_ROOT *root; - u64 vbo; size_t bit; - loff_t eod; int err = 0; struct inode *dir = file_inode(file); struct ntfs_inode *ni = ntfs_i(dir); struct super_block *sb = dir->i_sb; struct ntfs_sb_info *sbi = sb->s_fs_info; loff_t i_size = i_size_read(dir); - u32 pos = ctx->pos; + u64 pos = ctx->pos; u8 *name = NULL; struct indx_node *node = NULL; u8 index_bits = ni->dir.index_bits; + size_t max_bit = i_size >> ni->dir.index_bits; + loff_t eod = i_size + sbi->record_size; /* Name is a buffer of PATH_MAX length. */ static_assert(NTFS_NAME_LEN * 4 < PATH_MAX); - eod = i_size + sbi->record_size; + if (!pos) { + /* + * ni->dir.version increments each directory change. + * Save the initial value of ni->dir.version. + */ + file->private_data = (void *)ni->dir.version; + } - if (pos >= eod) - return 0; + if (pos >= eod) { + if (file->private_data == (void *)ni->dir.version) { + /* No changes since first readdir. */ + return 0; + } + + /* + * Handle directories that changed after the initial readdir(). + * + * Some user space code implements recursive removal like this instead + * of calling rmdir(2) directly: + * + * fd = opendir(path); + * while ((dent = readdir(fd))) + * unlinkat(dirfd(fd), dent->d_name, 0); + * closedir(fd); + * + * POSIX leaves unspecified what readdir() should return once the + * directory has been modified after opendir()/rewinddir(), so this + * pattern is not guaranteed to work on all filesystems or platforms. + * + * In ntfs3 the internal name tree may be reshaped while entries are + * being removed, so there is no stable anchor for continuing a + * single-pass walk based on the original readdir() order. + * + * In practice some widely used tools (for example certain rm(1) + * implementations) have used this readdir()/unlink() loop, and some + * filesystems behave in a way that effectively makes it work in the + * common case. + * + * The code below follows that practice and tries to provide + * "rmdir-like" behaviour for such callers on ntfs3, even though the + * situation is not strictly defined by the APIs. + * + * Apple documents the same readdir()/unlink() issue and a workaround + * for HFS file systems in: + * https://web.archive.org/web/20220122122948/https:/support.apple.com/kb/TA21420?locale=en_US + */ + ctx->pos = pos = 3; + file->private_data = (void *)ni->dir.version; + } if (!dir_emit_dots(file, ctx)) return 0; @@ -454,58 +498,58 @@ static int ntfs_readdir(struct file *file, struct dir_context *ctx) if (pos >= sbi->record_size) { bit = (pos - sbi->record_size) >> index_bits; } else { + /* + * Add each name from root in 'ctx'. + */ err = ntfs_read_hdr(sbi, ni, &root->ihdr, 0, pos, name, ctx); if (err) goto out; bit = 0; } - if (!i_size) { - ctx->pos = eod; - goto out; - } - - for (;;) { - vbo = (u64)bit << index_bits; - if (vbo >= i_size) { - ctx->pos = eod; - goto out; - } - + /* + * Enumerate indexes until the end of dir. + */ + for (; bit < max_bit; bit += 1) { + /* Get the next used index. */ err = indx_used_bit(&ni->dir, ni, &bit); if (err) goto out; if (bit == MINUS_ONE_T) { - ctx->pos = eod; - goto out; + /* no more used indexes. end of dir. */ + break; } - vbo = (u64)bit << index_bits; - if (vbo >= i_size) { + if (bit >= max_bit) { + /* Corrupted directory. */ err = -EINVAL; goto out; } - err = indx_read(&ni->dir, ni, bit << ni->dir.idx2vbn_bits, - &node); + err = indx_read_ra(&ni->dir, ni, bit << ni->dir.idx2vbn_bits, + &node, &file->f_ra); if (err) goto out; + /* + * Add each name from index in 'ctx'. + */ err = ntfs_read_hdr(sbi, ni, &node->index->ihdr, - vbo + sbi->record_size, pos, name, ctx); + ((u64)bit << index_bits) + sbi->record_size, + pos, name, ctx); if (err) goto out; - - bit += 1; } out: - kfree(name); put_indx_node(node); - if (err == 1) { + if (!err) { + /* End of directory. */ + ctx->pos = eod; + } else if (err == 1) { /* 'ctx' is full. */ err = 0; } else if (err == -ENOENT) { @@ -624,7 +668,7 @@ const struct file_operations ntfs_dir_operations = { .llseek = generic_file_llseek, .read = generic_read_dir, .iterate_shared = ntfs_readdir, - .fsync = generic_file_fsync, + .fsync = ntfs_file_fsync, .open = ntfs_file_open, .unlocked_ioctl = ntfs_ioctl, #ifdef CONFIG_COMPAT diff --git a/fs/ntfs3/file.c b/fs/ntfs3/file.c index 6cb4479072a6..ae8c47cac406 100644 --- a/fs/ntfs3/file.c +++ b/fs/ntfs3/file.c @@ -15,6 +15,7 @@ #include #include #include +#include #include "debug.h" #include "ntfs.h" @@ -26,6 +27,38 @@ */ #define NTFS3_IOC_SHUTDOWN _IOR('X', 125, __u32) +/* + * Helper for ntfs_should_use_dio. + */ +static u32 ntfs_dio_alignment(struct inode *inode) +{ + struct ntfs_inode *ni = ntfs_i(inode); + + if (is_resident(ni)) { + /* Check delalloc. */ + if (!ni->file.run_da.count) + return 0; + } + + /* In most cases this is bdev_logical_block_size(bdev). */ + return ni->mi.sbi->bdev_blocksize; +} + +/* + * Returns %true if the given DIO request should be attempted with DIO, or + * %false if it should fall back to buffered I/O. + */ +static bool ntfs_should_use_dio(struct kiocb *iocb, struct iov_iter *iter) +{ + struct inode *inode = file_inode(iocb->ki_filp); + u32 dio_align = ntfs_dio_alignment(inode); + + if (!dio_align) + return false; + + return IS_ALIGNED(iocb->ki_pos | iov_iter_alignment(iter), dio_align); +} + static int ntfs_ioctl_fitrim(struct ntfs_sb_info *sbi, unsigned long arg) { struct fstrim_range __user *user_range; @@ -186,13 +219,10 @@ int ntfs_getattr(struct mnt_idmap *idmap, const struct path *path, static int ntfs_extend_initialized_size(struct file *file, struct ntfs_inode *ni, - const loff_t valid, const loff_t new_valid) { struct inode *inode = &ni->vfs_inode; - struct address_space *mapping = inode->i_mapping; - struct ntfs_sb_info *sbi = inode->i_sb->s_fs_info; - loff_t pos = valid; + const loff_t valid = ni->i_valid; int err; if (valid >= new_valid) @@ -203,142 +233,41 @@ static int ntfs_extend_initialized_size(struct file *file, return 0; } - WARN_ON(is_compressed(ni)); - - for (;;) { - u32 zerofrom, len; - struct folio *folio; - u8 bits; - CLST vcn, lcn, clen; - - if (is_sparsed(ni)) { - bits = sbi->cluster_bits; - vcn = pos >> bits; - - err = attr_data_get_block(ni, vcn, 1, &lcn, &clen, NULL, - false); - if (err) - goto out; - - if (lcn == SPARSE_LCN) { - pos = ((loff_t)clen + vcn) << bits; - ni->i_valid = pos; - goto next; - } - } - - zerofrom = pos & (PAGE_SIZE - 1); - len = PAGE_SIZE - zerofrom; - - if (pos + len > new_valid) - len = new_valid - pos; - - err = ntfs_write_begin(NULL, mapping, pos, len, &folio, NULL); - if (err) - goto out; - - folio_zero_range(folio, zerofrom, folio_size(folio) - zerofrom); - - err = ntfs_write_end(NULL, mapping, pos, len, len, folio, NULL); - if (err < 0) - goto out; - pos += len; - -next: - if (pos >= new_valid) - break; - - balance_dirty_pages_ratelimited(mapping); - cond_resched(); + err = iomap_zero_range(inode, valid, new_valid - valid, NULL, + &ntfs_iomap_ops, &ntfs_iomap_folio_ops, NULL); + if (err) { + ni->i_valid = valid; + ntfs_inode_warn(inode, + "failed to extend initialized size to %llx.", + new_valid); + return err; } return 0; - -out: - ni->i_valid = valid; - ntfs_inode_warn(inode, "failed to extend initialized size to %llx.", - new_valid); - return err; } -/* - * ntfs_zero_range - Helper function for punch_hole. - * - * It zeroes a range [vbo, vbo_to). - */ -static int ntfs_zero_range(struct inode *inode, u64 vbo, u64 vbo_to) +static void ntfs_filemap_close(struct vm_area_struct *vma) { - int err = 0; - struct address_space *mapping = inode->i_mapping; - u32 blocksize = i_blocksize(inode); - pgoff_t idx = vbo >> PAGE_SHIFT; - u32 from = vbo & (PAGE_SIZE - 1); - pgoff_t idx_end = (vbo_to + PAGE_SIZE - 1) >> PAGE_SHIFT; - loff_t page_off; - struct buffer_head *head, *bh; - u32 bh_next, bh_off, to; - sector_t iblock; - struct folio *folio; - bool dirty = false; + struct inode *inode = file_inode(vma->vm_file); + struct ntfs_inode *ni = ntfs_i(inode); + u64 from = (u64)vma->vm_pgoff << PAGE_SHIFT; + u64 to = min_t(u64, i_size_read(inode), + from + vma->vm_end - vma->vm_start); - for (; idx < idx_end; idx += 1, from = 0) { - page_off = (loff_t)idx << PAGE_SHIFT; - to = (page_off + PAGE_SIZE) > vbo_to ? (vbo_to - page_off) : - PAGE_SIZE; - iblock = page_off >> inode->i_blkbits; - - folio = __filemap_get_folio( - mapping, idx, FGP_LOCK | FGP_ACCESSED | FGP_CREAT, - mapping_gfp_constraint(mapping, ~__GFP_FS)); - if (IS_ERR(folio)) - return PTR_ERR(folio); - - head = folio_buffers(folio); - if (!head) - head = create_empty_buffers(folio, blocksize, 0); - - bh = head; - bh_off = 0; - do { - bh_next = bh_off + blocksize; - - if (bh_next <= from || bh_off >= to) - continue; - - if (!buffer_mapped(bh)) { - ntfs_get_block(inode, iblock, bh, 0); - /* Unmapped? It's a hole - nothing to do. */ - if (!buffer_mapped(bh)) - continue; - } - - /* Ok, it's mapped. Make sure it's up-to-date. */ - if (folio_test_uptodate(folio)) - set_buffer_uptodate(bh); - else if (bh_read(bh, 0) < 0) { - err = -EIO; - folio_unlock(folio); - folio_put(folio); - goto out; - } - - mark_buffer_dirty(bh); - } while (bh_off = bh_next, iblock += 1, - head != (bh = bh->b_this_page)); - - folio_zero_segment(folio, from, to); - dirty = true; - - folio_unlock(folio); - folio_put(folio); - cond_resched(); - } -out: - if (dirty) + if (ni->i_valid < to) { + ni->i_valid = to; mark_inode_dirty(inode); - return err; + } } +/* Copy of generic_file_vm_ops. */ +static const struct vm_operations_struct ntfs_file_vm_ops = { + .close = ntfs_filemap_close, + .fault = filemap_fault, + .map_pages = filemap_map_pages, + .page_mkwrite = filemap_page_mkwrite, +}; + /* * ntfs_file_mmap_prepare - file_operations::mmap_prepare */ @@ -347,7 +276,6 @@ static int ntfs_file_mmap_prepare(struct vm_area_desc *desc) struct file *file = desc->file; struct inode *inode = file_inode(file); struct ntfs_inode *ni = ntfs_i(inode); - u64 from = ((u64)desc->pgoff << PAGE_SHIFT); bool rw = desc->vm_flags & VM_WRITE; int err; @@ -379,7 +307,8 @@ static int ntfs_file_mmap_prepare(struct vm_area_desc *desc) } if (rw) { - u64 to = min_t(loff_t, i_size_read(inode), + u64 from = (u64)desc->pgoff << PAGE_SHIFT; + u64 to = min_t(u64, i_size_read(inode), from + vma_desc_size(desc)); if (is_sparsed(ni)) { @@ -392,7 +321,8 @@ static int ntfs_file_mmap_prepare(struct vm_area_desc *desc) for (; vcn < end; vcn += len) { err = attr_data_get_block(ni, vcn, 1, &lcn, - &len, &new, true); + &len, &new, true, + NULL, false); if (err) goto out; } @@ -403,8 +333,7 @@ static int ntfs_file_mmap_prepare(struct vm_area_desc *desc) err = -EAGAIN; goto out; } - err = ntfs_extend_initialized_size(file, ni, - ni->i_valid, to); + err = ntfs_extend_initialized_size(file, ni, to); inode_unlock(inode); if (err) goto out; @@ -412,6 +341,8 @@ static int ntfs_file_mmap_prepare(struct vm_area_desc *desc) } err = generic_file_mmap_prepare(desc); + if (!err && rw) + desc->vm_ops = &ntfs_file_vm_ops; out: return err; } @@ -432,55 +363,23 @@ static int ntfs_extend(struct inode *inode, loff_t pos, size_t count, ntfs_set_state(ni->mi.sbi, NTFS_DIRTY_DIRTY); if (end > inode->i_size) { + /* + * Normal files: increase file size, allocate space. + * Sparse/Compressed: increase file size. No space allocated. + */ err = ntfs_set_size(inode, end); if (err) goto out; } if (extend_init && !is_compressed(ni)) { - err = ntfs_extend_initialized_size(file, ni, ni->i_valid, pos); + err = ntfs_extend_initialized_size(file, ni, pos); if (err) goto out; } else { err = 0; } - if (file && is_sparsed(ni)) { - /* - * This code optimizes large writes to sparse file. - * TODO: merge this fragment with fallocate fragment. - */ - struct ntfs_sb_info *sbi = ni->mi.sbi; - CLST vcn = pos >> sbi->cluster_bits; - CLST cend = bytes_to_cluster(sbi, end); - CLST cend_v = bytes_to_cluster(sbi, ni->i_valid); - CLST lcn, clen; - bool new; - - if (cend_v > cend) - cend_v = cend; - - /* - * Allocate and zero new clusters. - * Zeroing these clusters may be too long. - */ - for (; vcn < cend_v; vcn += clen) { - err = attr_data_get_block(ni, vcn, cend_v - vcn, &lcn, - &clen, &new, true); - if (err) - goto out; - } - /* - * Allocate but not zero new clusters. - */ - for (; vcn < cend; vcn += clen) { - err = attr_data_get_block(ni, vcn, cend - vcn, &lcn, - &clen, &new, false); - if (err) - goto out; - } - } - inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); mark_inode_dirty(inode); @@ -504,53 +403,37 @@ out: static int ntfs_truncate(struct inode *inode, loff_t new_size) { - struct super_block *sb = inode->i_sb; + int err; struct ntfs_inode *ni = ntfs_i(inode); - int err, dirty = 0; - u64 new_valid; - - if (!S_ISREG(inode->i_mode)) - return 0; - - if (is_compressed(ni)) { - if (ni->i_valid > new_size) - ni->i_valid = new_size; - } else { - err = block_truncate_page(inode->i_mapping, new_size, - ntfs_get_block); - if (err) - return err; - } - - new_valid = ntfs_up_block(sb, min_t(u64, ni->i_valid, new_size)); + u64 new_valid = min_t(u64, ni->i_valid, new_size); truncate_setsize(inode, new_size); ni_lock(ni); down_write(&ni->file.run_lock); - err = attr_set_size(ni, ATTR_DATA, NULL, 0, &ni->file.run, new_size, - &new_valid, ni->mi.sbi->options->prealloc, NULL); + err = attr_set_size_ex(ni, ATTR_DATA, NULL, 0, &ni->file.run, new_size, + &new_valid, ni->mi.sbi->options->prealloc, NULL, + false); up_write(&ni->file.run_lock); - if (new_valid < ni->i_valid) - ni->i_valid = new_valid; + ni->i_valid = new_valid; ni_unlock(ni); + if (err) + return err; + ni->std_fa |= FILE_ATTRIBUTE_ARCHIVE; inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); if (!IS_DIRSYNC(inode)) { - dirty = 1; + mark_inode_dirty(inode); } else { err = ntfs_sync_inode(inode); if (err) return err; } - if (dirty) - mark_inode_dirty(inode); - return 0; } @@ -623,7 +506,7 @@ static long ntfs_fallocate(struct file *file, int mode, loff_t vbo, loff_t len) if (mode & FALLOC_FL_PUNCH_HOLE) { u32 frame_size; - loff_t mask, vbo_a, end_a, tmp; + loff_t mask, vbo_a, end_a, tmp, from; err = filemap_write_and_wait_range(mapping, vbo_down, LLONG_MAX); @@ -643,24 +526,24 @@ static long ntfs_fallocate(struct file *file, int mode, loff_t vbo, loff_t len) /* Process not aligned punch. */ err = 0; + if (end > i_size) + end = i_size; mask = frame_size - 1; vbo_a = (vbo + mask) & ~mask; end_a = end & ~mask; tmp = min(vbo_a, end); - if (tmp > vbo) { - err = ntfs_zero_range(inode, vbo, tmp); + from = min_t(loff_t, ni->i_valid, vbo); + /* Zero head of punch. */ + if (tmp > from) { + err = iomap_zero_range(inode, from, tmp - from, NULL, + &ntfs_iomap_ops, + &ntfs_iomap_folio_ops, NULL); if (err) goto out; } - if (vbo < end_a && end_a < end) { - err = ntfs_zero_range(inode, end_a, end); - if (err) - goto out; - } - - /* Aligned punch_hole */ + /* Aligned punch_hole. Deallocate clusters. */ if (end_a > vbo_a) { ni_lock(ni); err = attr_punch_hole(ni, vbo_a, end_a - vbo_a, NULL); @@ -668,6 +551,15 @@ static long ntfs_fallocate(struct file *file, int mode, loff_t vbo, loff_t len) if (err) goto out; } + + /* Zero tail of punch. */ + if (vbo < end_a && end_a < end) { + err = iomap_zero_range(inode, end_a, end - end_a, NULL, + &ntfs_iomap_ops, + &ntfs_iomap_folio_ops, NULL); + if (err) + goto out; + } } else if (mode & FALLOC_FL_COLLAPSE_RANGE) { /* * Write tail of the last page before removed range since @@ -765,17 +657,26 @@ static long ntfs_fallocate(struct file *file, int mode, loff_t vbo, loff_t len) for (; vcn < cend_v; vcn += clen) { err = attr_data_get_block(ni, vcn, cend_v - vcn, &lcn, &clen, &new, - true); + true, NULL, false); if (err) goto out; } + + /* + * Moving up 'valid size'. + */ + err = ntfs_extend_initialized_size( + file, ni, (u64)cend_v << cluster_bits); + if (err) + goto out; + /* * Allocate but not zero new clusters. */ for (; vcn < cend; vcn += clen) { err = attr_data_get_block(ni, vcn, cend - vcn, &lcn, &clen, &new, - false); + false, NULL, false); if (err) goto out; } @@ -786,10 +687,11 @@ static long ntfs_fallocate(struct file *file, int mode, loff_t vbo, loff_t len) /* True - Keep preallocated. */ err = attr_set_size(ni, ATTR_DATA, NULL, 0, &ni->file.run, i_size, &ni->i_valid, - true, NULL); + true); ni_unlock(ni); if (err) goto out; + i_size_write(inode, i_size); } else if (new_size > i_size) { i_size_write(inode, new_size); } @@ -926,12 +828,18 @@ static ssize_t ntfs_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) struct file *file = iocb->ki_filp; struct inode *inode = file_inode(file); struct ntfs_inode *ni = ntfs_i(inode); + size_t bytes = iov_iter_count(iter); + loff_t valid, i_size, vbo, end; + unsigned int dio_flags; ssize_t err; err = check_read_restriction(inode); if (err) return err; + if (!bytes) + return 0; /* skip atime */ + if (is_compressed(ni)) { if (iocb->ki_flags & IOCB_DIRECT) { ntfs_inode_warn( @@ -942,17 +850,63 @@ static ssize_t ntfs_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) file->f_ra.ra_pages = 0; } - /* Check minimum alignment for dio. */ - if (iocb->ki_flags & IOCB_DIRECT) { - struct super_block *sb = inode->i_sb; - struct ntfs_sb_info *sbi = sb->s_fs_info; - if ((iocb->ki_pos | iov_iter_alignment(iter)) & - sbi->bdev_blocksize_mask) { - iocb->ki_flags &= ~IOCB_DIRECT; - } + /* Fallback to buffered I/O if the inode does not support direct I/O. */ + if (!(iocb->ki_flags & IOCB_DIRECT) || + !ntfs_should_use_dio(iocb, iter)) { + iocb->ki_flags &= ~IOCB_DIRECT; + return generic_file_read_iter(iocb, iter); } - return generic_file_read_iter(iocb, iter); + if (iocb->ki_flags & IOCB_NOWAIT) { + if (!inode_trylock_shared(inode)) + return -EAGAIN; + } else { + inode_lock_shared(inode); + } + + vbo = iocb->ki_pos; + end = vbo + bytes; + dio_flags = 0; + valid = ni->i_valid; + i_size = inode->i_size; + + if (vbo < valid) { + if (valid < end) { + /* read cross 'valid' size. */ + dio_flags |= IOMAP_DIO_FORCE_WAIT; + } + + if (ni->file.run_da.count) { + /* Direct I/O is not compatible with delalloc. */ + err = ni_allocate_da_blocks(ni); + if (err) + goto out; + } + + err = iomap_dio_rw(iocb, iter, &ntfs_iomap_ops, NULL, dio_flags, + NULL, 0); + + if (err <= 0) + goto out; + end = vbo + err; + if (valid < end) { + size_t to_zero = end - valid; + /* Fix iter. */ + iov_iter_revert(iter, to_zero); + iov_iter_zero(to_zero, iter); + } + } else if (vbo < i_size) { + if (end > i_size) + bytes = i_size - vbo; + iov_iter_zero(bytes, iter); + iocb->ki_pos += bytes; + err = bytes; + } + +out: + inode_unlock_shared(inode); + file_accessed(iocb->ki_filp); + return err; } /* @@ -996,7 +950,7 @@ static int ntfs_get_frame_pages(struct address_space *mapping, pgoff_t index, folio = __filemap_get_folio(mapping, index, FGP_LOCK | FGP_ACCESSED | FGP_CREAT, - gfp_mask); + gfp_mask | __GFP_ZERO); if (IS_ERR(folio)) { while (npages--) { folio = page_folio(pages[npages]); @@ -1073,7 +1027,7 @@ static ssize_t ntfs_compress_write(struct kiocb *iocb, struct iov_iter *from) off = valid & (frame_size - 1); err = attr_data_get_block(ni, frame << NTFS_LZNT_CUNIT, 1, &lcn, - &clen, NULL, false); + &clen, NULL, false, NULL, false); if (err) goto out; @@ -1265,6 +1219,9 @@ static int check_write_restriction(struct inode *inode) return -EOPNOTSUPP; } + if (unlikely(IS_IMMUTABLE(inode))) + return -EPERM; + return 0; } @@ -1276,8 +1233,7 @@ static ssize_t ntfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) struct file *file = iocb->ki_filp; struct inode *inode = file_inode(file); struct ntfs_inode *ni = ntfs_i(inode); - ssize_t ret; - int err; + ssize_t ret, err; if (!inode_trylock(inode)) { if (iocb->ki_flags & IOCB_NOWAIT) @@ -1315,15 +1271,75 @@ static ssize_t ntfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) if (ret) goto out; - ret = is_compressed(ni) ? ntfs_compress_write(iocb, from) : - __generic_file_write_iter(iocb, from); + if (is_compressed(ni)) { + ret = ntfs_compress_write(iocb, from); + goto out; + } + + /* Fallback to buffered I/O if the inode does not support direct I/O. */ + if (!(iocb->ki_flags & IOCB_DIRECT) || + !ntfs_should_use_dio(iocb, from)) { + iocb->ki_flags &= ~IOCB_DIRECT; + + ret = iomap_file_buffered_write(iocb, from, &ntfs_iomap_ops, + &ntfs_iomap_folio_ops, NULL); + inode_unlock(inode); + + if (likely(ret > 0)) + ret = generic_write_sync(iocb, ret); + + return ret; + } + + if (ni->file.run_da.count) { + /* Direct I/O is not compatible with delalloc. */ + ret = ni_allocate_da_blocks(ni); + if (ret) + goto out; + } + + ret = iomap_dio_rw(iocb, from, &ntfs_iomap_ops, NULL, 0, NULL, 0); + + if (ret == -ENOTBLK) { + /* Returns -ENOTBLK in case of a page invalidation failure for writes.*/ + /* The callers needs to fall back to buffered I/O in this case. */ + ret = 0; + } + + if (ret >= 0 && iov_iter_count(from)) { + loff_t offset = iocb->ki_pos, endbyte; + + iocb->ki_flags &= ~IOCB_DIRECT; + err = iomap_file_buffered_write(iocb, from, &ntfs_iomap_ops, + &ntfs_iomap_folio_ops, NULL); + if (err < 0) { + ret = err; + goto out; + } + + /* + * We need to ensure that the pages within the page cache for + * the range covered by this I/O are written to disk and + * invalidated. This is in attempt to preserve the expected + * direct I/O semantics in the case we fallback to buffered I/O + * to complete off the I/O request. + */ + ret += err; + endbyte = offset + err - 1; + err = filemap_write_and_wait_range(inode->i_mapping, offset, + endbyte); + if (err) { + ret = err; + goto out; + } + + invalidate_mapping_pages(inode->i_mapping, offset >> PAGE_SHIFT, + endbyte >> PAGE_SHIFT); + } out: inode_unlock(inode); - if (ret > 0) - ret = generic_write_sync(iocb, ret); - return ret; } @@ -1362,39 +1378,49 @@ int ntfs_file_open(struct inode *inode, struct file *file) #endif } + file->f_mode |= FMODE_NOWAIT | FMODE_CAN_ODIRECT; + return generic_file_open(inode, file); } /* * ntfs_file_release - file_operations::release + * + * Called when an inode is released. Note that this is different + * from ntfs_file_open: open gets called at every open, but release + * gets called only when /all/ the files are closed. */ static int ntfs_file_release(struct inode *inode, struct file *file) { - struct ntfs_inode *ni = ntfs_i(inode); - struct ntfs_sb_info *sbi = ni->mi.sbi; - int err = 0; + int err; + struct ntfs_inode *ni; - /* If we are last writer on the inode, drop the block reservation. */ - if (sbi->options->prealloc && - ((file->f_mode & FMODE_WRITE) && - atomic_read(&inode->i_writecount) == 1) - /* - * The only file when inode->i_fop = &ntfs_file_operations and - * init_rwsem(&ni->file.run_lock) is not called explicitly is MFT. - * - * Add additional check here. - */ - && inode->i_ino != MFT_REC_MFT) { + if (!(file->f_mode & FMODE_WRITE) || + atomic_read(&inode->i_writecount) != 1 || + inode->i_ino == MFT_REC_MFT) { + return 0; + } + + /* Close the last writer on the inode. */ + ni = ntfs_i(inode); + + /* Allocate delayed blocks (clusters). */ + err = ni_allocate_da_blocks(ni); + if (err) + goto out; + + if (ni->mi.sbi->options->prealloc) { ni_lock(ni); down_write(&ni->file.run_lock); + /* Deallocate preallocated. */ err = attr_set_size(ni, ATTR_DATA, NULL, 0, &ni->file.run, - i_size_read(inode), &ni->i_valid, false, - NULL); + inode->i_size, &ni->i_valid, false); up_write(&ni->file.run_lock); ni_unlock(ni); } +out: return err; } @@ -1411,16 +1437,30 @@ int ntfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, if (unlikely(is_bad_ni(ni))) return -EINVAL; - err = fiemap_prep(inode, fieinfo, start, &len, ~FIEMAP_FLAG_XATTR); - if (err) - return err; + if (is_compressed(ni)) { + /* Unfortunately cp -r incorrectly treats compressed clusters. */ + ntfs_inode_warn(inode, + "fiemap is not supported for compressed file"); + return -EOPNOTSUPP; + } - ni_lock(ni); + if (S_ISDIR(inode->i_mode)) { + /* TODO: add support for dirs (ATTR_ALLOC). */ + ntfs_inode_warn(inode, + "fiemap is not supported for directories"); + return -EOPNOTSUPP; + } - err = ni_fiemap(ni, fieinfo, start, len); + if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) { + ntfs_inode_warn(inode, "fiemap(xattr) is not supported"); + return -EOPNOTSUPP; + } - ni_unlock(ni); + inode_lock_shared(inode); + err = iomap_fiemap(inode, fieinfo, start, len, &ntfs_iomap_ops); + + inode_unlock_shared(inode); return err; } @@ -1444,13 +1484,62 @@ static ssize_t ntfs_file_splice_write(struct pipe_inode_info *pipe, /* * ntfs_file_fsync - file_operations::fsync */ -static int ntfs_file_fsync(struct file *file, loff_t start, loff_t end, int datasync) +int ntfs_file_fsync(struct file *file, loff_t start, loff_t end, int datasync) { struct inode *inode = file_inode(file); - if (unlikely(ntfs3_forced_shutdown(inode->i_sb))) + struct super_block *sb = inode->i_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + int err, ret; + + if (unlikely(ntfs3_forced_shutdown(sb))) return -EIO; - return generic_file_fsync(file, start, end, datasync); + ret = file_write_and_wait_range(file, start, end); + if (ret) + return ret; + + ret = write_inode_now(inode, !datasync); + + if (!ret) { + ret = ni_write_parents(ntfs_i(inode), !datasync); + } + + if (!ret) { + ntfs_set_state(sbi, NTFS_DIRTY_CLEAR); + ntfs_update_mftmirr(sbi); + } + + err = sync_blockdev(sb->s_bdev); + if (unlikely(err && !ret)) + ret = err; + if (!ret) + blkdev_issue_flush(sb->s_bdev); + return ret; +} + +/* + * ntfs_llseek - file_operations::llseek + */ +static loff_t ntfs_llseek(struct file *file, loff_t offset, int whence) +{ + struct inode *inode = file->f_mapping->host; + struct ntfs_inode *ni = ntfs_i(inode); + loff_t maxbytes = ntfs_get_maxbytes(ni); + loff_t ret; + + if (whence == SEEK_DATA || whence == SEEK_HOLE) { + inode_lock_shared(inode); + /* Scan file for hole or data. */ + ret = ni_seek_data_or_hole(ni, offset, whence == SEEK_DATA); + inode_unlock_shared(inode); + + if (ret >= 0) + ret = vfs_setpos(file, ret, maxbytes); + } else { + ret = generic_file_llseek_size(file, offset, whence, maxbytes, + i_size_read(inode)); + } + return ret; } // clang-format off @@ -1464,7 +1553,7 @@ const struct inode_operations ntfs_file_inode_operations = { }; const struct file_operations ntfs_file_operations = { - .llseek = generic_file_llseek, + .llseek = ntfs_llseek, .read_iter = ntfs_file_read_iter, .write_iter = ntfs_file_write_iter, .unlocked_ioctl = ntfs_ioctl, diff --git a/fs/ntfs3/frecord.c b/fs/ntfs3/frecord.c index 641ddaf8d4a0..bd0fa481e4b3 100644 --- a/fs/ntfs3/frecord.c +++ b/fs/ntfs3/frecord.c @@ -123,6 +123,8 @@ void ni_clear(struct ntfs_inode *ni) indx_clear(&ni->dir); else { run_close(&ni->file.run); + ntfs_sub_da(ni->mi.sbi, run_len(&ni->file.run_da)); + run_close(&ni->file.run_da); #ifdef CONFIG_NTFS3_LZX_XPRESS if (ni->file.offs_folio) { /* On-demand allocated page for offsets. */ @@ -1850,183 +1852,11 @@ enum REPARSE_SIGN ni_parse_reparse(struct ntfs_inode *ni, struct ATTRIB *attr, return REPARSE_LINK; } -/* - * ni_fiemap - Helper for file_fiemap(). - * - * Assumed ni_lock. - * TODO: Less aggressive locks. - */ -int ni_fiemap(struct ntfs_inode *ni, struct fiemap_extent_info *fieinfo, - __u64 vbo, __u64 len) -{ - int err = 0; - struct ntfs_sb_info *sbi = ni->mi.sbi; - u8 cluster_bits = sbi->cluster_bits; - struct runs_tree run; - struct ATTRIB *attr; - CLST vcn = vbo >> cluster_bits; - CLST lcn, clen; - u64 valid = ni->i_valid; - u64 lbo, bytes; - u64 end, alloc_size; - size_t idx = -1; - u32 flags; - bool ok; - - run_init(&run); - if (S_ISDIR(ni->vfs_inode.i_mode)) { - attr = ni_find_attr(ni, NULL, NULL, ATTR_ALLOC, I30_NAME, - ARRAY_SIZE(I30_NAME), NULL, NULL); - } else { - attr = ni_find_attr(ni, NULL, NULL, ATTR_DATA, NULL, 0, NULL, - NULL); - if (!attr) { - err = -EINVAL; - goto out; - } - if (is_attr_compressed(attr)) { - /* Unfortunately cp -r incorrectly treats compressed clusters. */ - err = -EOPNOTSUPP; - ntfs_inode_warn( - &ni->vfs_inode, - "fiemap is not supported for compressed file (cp -r)"); - goto out; - } - } - - if (!attr || !attr->non_res) { - err = fiemap_fill_next_extent( - fieinfo, 0, 0, - attr ? le32_to_cpu(attr->res.data_size) : 0, - FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_LAST | - FIEMAP_EXTENT_MERGED); - goto out; - } - - end = vbo + len; - alloc_size = le64_to_cpu(attr->nres.alloc_size); - if (end > alloc_size) - end = alloc_size; - - while (vbo < end) { - if (idx == -1) { - ok = run_lookup_entry(&run, vcn, &lcn, &clen, &idx); - } else { - CLST vcn_next = vcn; - - ok = run_get_entry(&run, ++idx, &vcn, &lcn, &clen) && - vcn == vcn_next; - if (!ok) - vcn = vcn_next; - } - - if (!ok) { - err = attr_load_runs_vcn(ni, attr->type, - attr_name(attr), - attr->name_len, &run, vcn); - - if (err) - break; - - ok = run_lookup_entry(&run, vcn, &lcn, &clen, &idx); - - if (!ok) { - err = -EINVAL; - break; - } - } - - if (!clen) { - err = -EINVAL; // ? - break; - } - - if (lcn == SPARSE_LCN) { - vcn += clen; - vbo = (u64)vcn << cluster_bits; - continue; - } - - flags = FIEMAP_EXTENT_MERGED; - if (S_ISDIR(ni->vfs_inode.i_mode)) { - ; - } else if (is_attr_compressed(attr)) { - CLST clst_data; - - err = attr_is_frame_compressed(ni, attr, - vcn >> attr->nres.c_unit, - &clst_data, &run); - if (err) - break; - if (clst_data < NTFS_LZNT_CLUSTERS) - flags |= FIEMAP_EXTENT_ENCODED; - } else if (is_attr_encrypted(attr)) { - flags |= FIEMAP_EXTENT_DATA_ENCRYPTED; - } - - vbo = (u64)vcn << cluster_bits; - bytes = (u64)clen << cluster_bits; - lbo = (u64)lcn << cluster_bits; - - vcn += clen; - - if (vbo + bytes >= end) - bytes = end - vbo; - - if (vbo + bytes <= valid) { - ; - } else if (vbo >= valid) { - flags |= FIEMAP_EXTENT_UNWRITTEN; - } else { - /* vbo < valid && valid < vbo + bytes */ - u64 dlen = valid - vbo; - - if (vbo + dlen >= end) - flags |= FIEMAP_EXTENT_LAST; - - err = fiemap_fill_next_extent(fieinfo, vbo, lbo, dlen, - flags); - - if (err < 0) - break; - if (err == 1) { - err = 0; - break; - } - - vbo = valid; - bytes -= dlen; - if (!bytes) - continue; - - lbo += dlen; - flags |= FIEMAP_EXTENT_UNWRITTEN; - } - - if (vbo + bytes >= end) - flags |= FIEMAP_EXTENT_LAST; - - err = fiemap_fill_next_extent(fieinfo, vbo, lbo, bytes, flags); - if (err < 0) - break; - if (err == 1) { - err = 0; - break; - } - - vbo += bytes; - } - -out: - run_close(&run); - return err; -} - static struct page *ntfs_lock_new_page(struct address_space *mapping, - pgoff_t index, gfp_t gfp) + pgoff_t index, gfp_t gfp) { - struct folio *folio = __filemap_get_folio(mapping, index, - FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp); + struct folio *folio = __filemap_get_folio( + mapping, index, FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp); struct page *page; if (IS_ERR(folio)) @@ -2046,18 +1876,18 @@ static struct page *ntfs_lock_new_page(struct address_space *mapping, } /* - * ni_readpage_cmpr + * ni_read_folio_cmpr * * When decompressing, we typically obtain more than one page per reference. * We inject the additional pages into the page cache. */ -int ni_readpage_cmpr(struct ntfs_inode *ni, struct folio *folio) +int ni_read_folio_cmpr(struct ntfs_inode *ni, struct folio *folio) { int err; struct ntfs_sb_info *sbi = ni->mi.sbi; struct address_space *mapping = folio->mapping; - pgoff_t index = folio->index; - u64 frame_vbo, vbo = (u64)index << PAGE_SHIFT; + pgoff_t index; + u64 frame_vbo, vbo = folio_pos(folio); struct page **pages = NULL; /* Array of at most 16 pages. stack? */ u8 frame_bits; CLST frame; @@ -2107,7 +1937,9 @@ int ni_readpage_cmpr(struct ntfs_inode *ni, struct folio *folio) pages[i] = pg; } + ni_lock(ni); err = ni_read_frame(ni, frame_vbo, pages, pages_per_frame, 0); + ni_unlock(ni); out1: for (i = 0; i < pages_per_frame; i++) { @@ -2184,7 +2016,8 @@ int ni_decompress_file(struct ntfs_inode *ni) for (vcn = vbo >> sbi->cluster_bits; vcn < end; vcn += clen) { err = attr_data_get_block(ni, vcn, cend - vcn, &lcn, - &clen, &new, false); + &clen, &new, false, NULL, + false); if (err) goto out; } @@ -2405,7 +2238,7 @@ int ni_read_frame(struct ntfs_inode *ni, u64 frame_vbo, struct page **pages, struct runs_tree *run = &ni->file.run; u64 valid_size = ni->i_valid; u64 vbo_disk; - size_t unc_size; + size_t unc_size = 0; u32 frame_size, i, ondisk_size; struct page *pg; struct ATTRIB *attr; @@ -3001,6 +2834,134 @@ bool ni_is_dirty(struct inode *inode) return false; } +/* + * ni_seek_data_or_hole + * + * Helper function for ntfs_llseek( SEEK_DATA/SEEK_HOLE ) + */ +loff_t ni_seek_data_or_hole(struct ntfs_inode *ni, loff_t offset, bool data) +{ + int err; + u8 cluster_bits = ni->mi.sbi->cluster_bits; + CLST vcn, lcn, clen; + loff_t vbo; + + /* Enumerate all fragments. */ + for (vcn = offset >> cluster_bits;; vcn += clen) { + err = attr_data_get_block(ni, vcn, 1, &lcn, &clen, NULL, false, + NULL, false); + if (err) { + return err; + } + + if (lcn == RESIDENT_LCN) { + /* clen - resident size in bytes. clen == ni->vfs_inode.i_size */ + if (offset >= clen) { + /* check eof. */ + return -ENXIO; + } + + if (data) { + return offset; + } + + return clen; + } + + if (lcn == EOF_LCN) { + if (data) { + return -ENXIO; + } + + /* implicit hole at the end of file. */ + return ni->vfs_inode.i_size; + } + + if (data) { + /* + * Adjust the file offset to the next location in the file greater than + * or equal to offset containing data. If offset points to data, then + * the file offset is set to offset. + */ + if (lcn != SPARSE_LCN) { + vbo = (u64)vcn << cluster_bits; + return max(vbo, offset); + } + } else { + /* + * Adjust the file offset to the next hole in the file greater than or + * equal to offset. If offset points into the middle of a hole, then the + * file offset is set to offset. If there is no hole past offset, then the + * file offset is adjusted to the end of the file + * (i.e., there is an implicit hole at the end of any file). + */ + if (lcn == SPARSE_LCN && + /* native compression hole begins at aligned vcn. */ + (!(ni->std_fa & FILE_ATTRIBUTE_COMPRESSED) || + !(vcn & (NTFS_LZNT_CLUSTERS - 1)))) { + vbo = (u64)vcn << cluster_bits; + return max(vbo, offset); + } + } + + if (!clen) { + /* Corrupted file. */ + return -EINVAL; + } + } +} + +/* + * ni_write_parents + * + * Helper function for ntfs_file_fsync. + */ +int ni_write_parents(struct ntfs_inode *ni, int sync) +{ + int err = 0; + struct ATTRIB *attr = NULL; + struct ATTR_LIST_ENTRY *le = NULL; + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct super_block *sb = sbi->sb; + + while ((attr = ni_find_attr(ni, attr, &le, ATTR_NAME, NULL, 0, NULL, + NULL))) { + struct inode *dir; + struct ATTR_FILE_NAME *fname; + + fname = resident_data_ex(attr, SIZEOF_ATTRIBUTE_FILENAME); + if (!fname) + continue; + + /* Check simple case when parent inode equals current inode. */ + if (ino_get(&fname->home) == ni->vfs_inode.i_ino) { + if (MFT_REC_ROOT != ni->vfs_inode.i_ino) { + ntfs_set_state(sbi, NTFS_DIRTY_ERROR); + err = -EINVAL; + } + continue; + } + + dir = ntfs_iget5(sb, &fname->home, NULL); + if (IS_ERR(dir)) { + ntfs_inode_warn( + &ni->vfs_inode, + "failed to open parent directory r=%lx to write", + (long)ino_get(&fname->home)); + continue; + } + + if (!is_bad_inode(dir)) { + int err2 = write_inode_now(dir, sync); + if (!err) + err = err2; + } + iput(dir); + } + + return err; +} + /* * ni_update_parent * @@ -3277,3 +3238,62 @@ out: return 0; } + +/* + * Force to allocate all delay allocated clusters. + */ +int ni_allocate_da_blocks(struct ntfs_inode *ni) +{ + int err; + + ni_lock(ni); + down_write(&ni->file.run_lock); + + err = ni_allocate_da_blocks_locked(ni); + + up_write(&ni->file.run_lock); + ni_unlock(ni); + + return err; +} + +/* + * Force to allocate all delay allocated clusters. + */ +int ni_allocate_da_blocks_locked(struct ntfs_inode *ni) +{ + int err; + + if (!ni->file.run_da.count) + return 0; + + if (is_sparsed(ni)) { + CLST vcn, lcn, clen, alen; + bool new; + + /* + * Sparse file allocates clusters in 'attr_data_get_block_locked' + */ + while (run_get_entry(&ni->file.run_da, 0, &vcn, &lcn, &clen)) { + /* TODO: zero=true? */ + err = attr_data_get_block_locked(ni, vcn, clen, &lcn, + &alen, &new, true, + NULL, true); + if (err) + break; + if (!new) { + err = -EINVAL; + break; + } + } + } else { + /* + * Normal file allocates clusters in 'attr_set_size' + */ + err = attr_set_size_ex(ni, ATTR_DATA, NULL, 0, &ni->file.run, + ni->vfs_inode.i_size, &ni->i_valid, + false, NULL, true); + } + + return err; +} diff --git a/fs/ntfs3/fslog.c b/fs/ntfs3/fslog.c index 38934e6978ec..10863c83c315 100644 --- a/fs/ntfs3/fslog.c +++ b/fs/ntfs3/fslog.c @@ -1074,6 +1074,8 @@ struct ntfs_log { u32 client_undo_commit; struct restart_info rst_info, rst_info2; + + struct file_ra_state read_ahead; }; static inline u32 lsn_to_vbo(struct ntfs_log *log, const u64 lsn) @@ -1164,8 +1166,8 @@ static int read_log_page(struct ntfs_log *log, u32 vbo, page_buf = page_off ? log->one_page_buf : *buffer; - err = ntfs_read_run_nb(ni->mi.sbi, &ni->file.run, page_vbo, page_buf, - log->page_size, NULL); + err = ntfs_read_run_nb_ra(ni->mi.sbi, &ni->file.run, page_vbo, page_buf, + log->page_size, NULL, &log->read_ahead); if (err) goto out; @@ -3028,6 +3030,26 @@ static struct ATTRIB *attr_create_nonres_log(struct ntfs_sb_info *sbi, return attr; } +/* + * update_oa_attr - Synchronize OpenAttr's attribute pointer with modified attribute + * @oa2: OpenAttr structure in memory that needs to be updated + * @attr: Modified attribute from MFT record to duplicate + * + * Returns true on success, false on allocation failure. + */ +static bool update_oa_attr(struct OpenAttr *oa2, struct ATTRIB *attr) +{ + void *p2; + + p2 = kmemdup(attr, le32_to_cpu(attr->size), GFP_NOFS); + if (p2) { + kfree(oa2->attr); + oa2->attr = p2; + return true; + } + return false; +} + /* * do_action - Common routine for the Redo and Undo Passes. * @rlsn: If it is NULL then undo. @@ -3251,15 +3273,8 @@ skip_load_parent: le16_add_cpu(&rec->hard_links, 1); oa2 = find_loaded_attr(log, attr, rno_base); - if (oa2) { - void *p2 = kmemdup(attr, le32_to_cpu(attr->size), - GFP_NOFS); - if (p2) { - // run_close(oa2->run1); - kfree(oa2->attr); - oa2->attr = p2; - } - } + if (oa2) + update_oa_attr(oa2, attr); mi->dirty = true; break; @@ -3318,16 +3333,8 @@ move_data: memmove(Add2Ptr(attr, aoff), data, dlen); oa2 = find_loaded_attr(log, attr, rno_base); - if (oa2) { - void *p2 = kmemdup(attr, le32_to_cpu(attr->size), - GFP_NOFS); - if (p2) { - // run_close(&oa2->run0); - oa2->run1 = &oa2->run0; - kfree(oa2->attr); - oa2->attr = p2; - } - } + if (oa2 && update_oa_attr(oa2, attr)) + oa2->run1 = &oa2->run0; mi->dirty = true; break; @@ -3377,14 +3384,9 @@ move_data: attr->nres.total_size = new_sz->total_size; oa2 = find_loaded_attr(log, attr, rno_base); - if (oa2) { - void *p2 = kmemdup(attr, le32_to_cpu(attr->size), - GFP_NOFS); - if (p2) { - kfree(oa2->attr); - oa2->attr = p2; - } - } + if (oa2) + update_oa_attr(oa2, attr); + mi->dirty = true; break; @@ -3429,6 +3431,9 @@ move_data: e1 = Add2Ptr(attr, le16_to_cpu(lrh->attr_off)); esize = le16_to_cpu(e1->size); + if (PtrOffset(e1, Add2Ptr(hdr, used)) < esize) + goto dirty_vol; + e2 = Add2Ptr(e1, esize); memmove(e1, e2, PtrOffset(e2, Add2Ptr(hdr, used))); @@ -5128,7 +5133,7 @@ commit_undo: undo_action_done: - ntfs_update_mftmirr(sbi, 0); + ntfs_update_mftmirr(sbi); sbi->flags &= ~NTFS_FLAGS_NEED_REPLAY; diff --git a/fs/ntfs3/fsntfs.c b/fs/ntfs3/fsntfs.c index bd67ba7b5015..0df2aa81d884 100644 --- a/fs/ntfs3/fsntfs.c +++ b/fs/ntfs3/fsntfs.c @@ -445,36 +445,59 @@ up_write: } /* - * ntfs_check_for_free_space + * ntfs_check_free_space * * Check if it is possible to allocate 'clen' clusters and 'mlen' Mft records */ -bool ntfs_check_for_free_space(struct ntfs_sb_info *sbi, CLST clen, CLST mlen) +bool ntfs_check_free_space(struct ntfs_sb_info *sbi, CLST clen, CLST mlen, + bool da) { size_t free, zlen, avail; struct wnd_bitmap *wnd; + CLST da_clusters = ntfs_get_da(sbi); wnd = &sbi->used.bitmap; down_read_nested(&wnd->rw_lock, BITMAP_MUTEX_CLUSTERS); free = wnd_zeroes(wnd); + + if (free >= da_clusters) { + free -= da_clusters; + } else { + free = 0; + } + zlen = min_t(size_t, NTFS_MIN_MFT_ZONE, wnd_zone_len(wnd)); up_read(&wnd->rw_lock); - if (free < zlen + clen) + if (free < zlen + clen) { return false; + } avail = free - (zlen + clen); - wnd = &sbi->mft.bitmap; - down_read_nested(&wnd->rw_lock, BITMAP_MUTEX_MFT); - free = wnd_zeroes(wnd); - zlen = wnd_zone_len(wnd); - up_read(&wnd->rw_lock); + /* + * When delalloc is active then keep in mind some reserved space. + * The worst case: 1 mft record per each ~500 clusters. + */ + if (da) { + /* 1 mft record per each 1024 clusters. */ + mlen += da_clusters >> 10; + } - if (free >= zlen + mlen) - return true; + if (mlen || !avail) { + wnd = &sbi->mft.bitmap; + down_read_nested(&wnd->rw_lock, BITMAP_MUTEX_MFT); + free = wnd_zeroes(wnd); + zlen = wnd_zone_len(wnd); + up_read(&wnd->rw_lock); - return avail >= bytes_to_cluster(sbi, mlen << sbi->record_bits); + if (free < zlen + mlen && + avail < bytes_to_cluster(sbi, mlen << sbi->record_bits)) { + return false; + } + } + + return true; } /* @@ -509,8 +532,8 @@ static int ntfs_extend_mft(struct ntfs_sb_info *sbi) /* Step 1: Resize $MFT::DATA. */ down_write(&ni->file.run_lock); - err = attr_set_size(ni, ATTR_DATA, NULL, 0, &ni->file.run, - new_mft_bytes, NULL, false, &attr); + err = attr_set_size_ex(ni, ATTR_DATA, NULL, 0, &ni->file.run, + new_mft_bytes, NULL, false, &attr, false); if (err) { up_write(&ni->file.run_lock); @@ -525,7 +548,7 @@ static int ntfs_extend_mft(struct ntfs_sb_info *sbi) new_bitmap_bytes = ntfs3_bitmap_size(new_mft_total); err = attr_set_size(ni, ATTR_BITMAP, NULL, 0, &sbi->mft.bitmap.run, - new_bitmap_bytes, &new_bitmap_bytes, true, NULL); + new_bitmap_bytes, &new_bitmap_bytes, true); /* Refresh MFT Zone if necessary. */ down_write_nested(&sbi->used.bitmap.rw_lock, BITMAP_MUTEX_CLUSTERS); @@ -843,9 +866,8 @@ int ntfs_refresh_zone(struct ntfs_sb_info *sbi) /* * ntfs_update_mftmirr - Update $MFTMirr data. */ -void ntfs_update_mftmirr(struct ntfs_sb_info *sbi, int wait) +void ntfs_update_mftmirr(struct ntfs_sb_info *sbi) { - int err; struct super_block *sb = sbi->sb; u32 blocksize, bytes; sector_t block1, block2; @@ -875,9 +897,7 @@ void ntfs_update_mftmirr(struct ntfs_sb_info *sbi, int wait) return; } - if (buffer_locked(bh2)) - __wait_on_buffer(bh2); - + wait_on_buffer(bh2); lock_buffer(bh2); memcpy(bh2->b_data, bh1->b_data, blocksize); set_buffer_uptodate(bh2); @@ -886,12 +906,7 @@ void ntfs_update_mftmirr(struct ntfs_sb_info *sbi, int wait) put_bh(bh1); bh1 = NULL; - - err = wait ? sync_dirty_buffer(bh2) : 0; - put_bh(bh2); - if (err) - return; } sbi->flags &= ~NTFS_FLAGS_MFTMIRR; @@ -1069,9 +1084,7 @@ int ntfs_sb_write(struct super_block *sb, u64 lbo, size_t bytes, return -ENOMEM; } - if (buffer_locked(bh)) - __wait_on_buffer(bh); - + wait_on_buffer(bh); lock_buffer(bh); if (buf) { memcpy(bh->b_data + off, buf, op); @@ -1168,11 +1181,13 @@ struct buffer_head *ntfs_bread_run(struct ntfs_sb_info *sbi, return ntfs_bread(sb, lbo >> sb->s_blocksize_bits); } -int ntfs_read_run_nb(struct ntfs_sb_info *sbi, const struct runs_tree *run, - u64 vbo, void *buf, u32 bytes, struct ntfs_buffers *nb) +int ntfs_read_run_nb_ra(struct ntfs_sb_info *sbi, const struct runs_tree *run, + u64 vbo, void *buf, u32 bytes, struct ntfs_buffers *nb, + struct file_ra_state *ra) { int err; struct super_block *sb = sbi->sb; + struct address_space *mapping = sb->s_bdev->bd_mapping; u32 blocksize = sb->s_blocksize; u8 cluster_bits = sbi->cluster_bits; u32 off = vbo & sbi->cluster_mask; @@ -1212,10 +1227,22 @@ int ntfs_read_run_nb(struct ntfs_sb_info *sbi, const struct runs_tree *run, nb->bytes = bytes; } + if (ra && !ra->ra_pages) + file_ra_state_init(ra, mapping); + for (;;) { u32 len32 = len >= bytes ? bytes : len; sector_t block = lbo >> sb->s_blocksize_bits; + if (ra) { + pgoff_t index = lbo >> PAGE_SHIFT; + if (!ra_has_index(ra, index)) { + page_cache_sync_readahead(mapping, ra, NULL, + index, 1); + ra->prev_pos = (loff_t)index << PAGE_SHIFT; + } + } + do { u32 op = blocksize - off; @@ -1252,6 +1279,12 @@ int ntfs_read_run_nb(struct ntfs_sb_info *sbi, const struct runs_tree *run, } while (len32); + if (!run) { + err = -EINVAL; + goto out; + } + + /* Get next fragment to read. */ vcn_next = vcn + clen; if (!run_get_entry(run, ++idx, &vcn, &lcn, &clen) || vcn != vcn_next) { @@ -1286,11 +1319,11 @@ out: * * Return: < 0 if error, 0 if ok, -E_NTFS_FIXUP if need to update fixups. */ -int ntfs_read_bh(struct ntfs_sb_info *sbi, const struct runs_tree *run, u64 vbo, - struct NTFS_RECORD_HEADER *rhdr, u32 bytes, - struct ntfs_buffers *nb) +int ntfs_read_bh_ra(struct ntfs_sb_info *sbi, const struct runs_tree *run, + u64 vbo, struct NTFS_RECORD_HEADER *rhdr, u32 bytes, + struct ntfs_buffers *nb, struct file_ra_state *ra) { - int err = ntfs_read_run_nb(sbi, run, vbo, rhdr, bytes, nb); + int err = ntfs_read_run_nb_ra(sbi, run, vbo, rhdr, bytes, nb, ra); if (err) return err; @@ -1347,12 +1380,9 @@ int ntfs_get_bh(struct ntfs_sb_info *sbi, const struct runs_tree *run, u64 vbo, err = -ENOMEM; goto out; } - if (buffer_locked(bh)) - __wait_on_buffer(bh); - + wait_on_buffer(bh); lock_buffer(bh); - if (!buffer_uptodate(bh)) - { + if (!buffer_uptodate(bh)) { memset(bh->b_data, 0, blocksize); set_buffer_uptodate(bh); } @@ -1427,9 +1457,7 @@ int ntfs_write_bh(struct ntfs_sb_info *sbi, struct NTFS_RECORD_HEADER *rhdr, if (op > bytes) op = bytes; - if (buffer_locked(bh)) - __wait_on_buffer(bh); - + wait_on_buffer(bh); lock_buffer(bh); bh_data = bh->b_data + off; @@ -2186,7 +2214,7 @@ int ntfs_insert_security(struct ntfs_sb_info *sbi, if (new_sds_size > ni->vfs_inode.i_size) { err = attr_set_size(ni, ATTR_DATA, SDS_NAME, ARRAY_SIZE(SDS_NAME), &ni->file.run, - new_sds_size, &new_sds_size, false, NULL); + new_sds_size, &new_sds_size, false); if (err) goto out; } diff --git a/fs/ntfs3/index.c b/fs/ntfs3/index.c index 7157cfd70fdc..2416c61050f1 100644 --- a/fs/ntfs3/index.c +++ b/fs/ntfs3/index.c @@ -252,9 +252,7 @@ static int bmp_buf_get(struct ntfs_index *indx, struct ntfs_inode *ni, bbuf->bh = bh; - if (buffer_locked(bh)) - __wait_on_buffer(bh); - + wait_on_buffer(bh); lock_buffer(bh); sb = sbi->sb; @@ -1028,17 +1026,18 @@ static int indx_write(struct ntfs_index *indx, struct ntfs_inode *ni, } /* - * indx_read + * indx_read_ra * * If ntfs_readdir calls this function * inode is shared locked and no ni_lock. * Use rw_semaphore for read/write access to alloc_run. */ -int indx_read(struct ntfs_index *indx, struct ntfs_inode *ni, CLST vbn, - struct indx_node **node) +int indx_read_ra(struct ntfs_index *indx, struct ntfs_inode *ni, CLST vbn, + struct indx_node **node, struct file_ra_state *ra) { int err; struct INDEX_BUFFER *ib; + struct ntfs_sb_info *sbi = ni->mi.sbi; struct runs_tree *run = &indx->alloc_run; struct rw_semaphore *lock = &indx->run_lock; u64 vbo = (u64)vbn << indx->vbn2vbo_bits; @@ -1064,7 +1063,7 @@ int indx_read(struct ntfs_index *indx, struct ntfs_inode *ni, CLST vbn, } down_read(lock); - err = ntfs_read_bh(ni->mi.sbi, run, vbo, &ib->rhdr, bytes, &in->nb); + err = ntfs_read_bh_ra(sbi, run, vbo, &ib->rhdr, bytes, &in->nb, ra); up_read(lock); if (!err) goto ok; @@ -1084,7 +1083,7 @@ int indx_read(struct ntfs_index *indx, struct ntfs_inode *ni, CLST vbn, goto out; down_read(lock); - err = ntfs_read_bh(ni->mi.sbi, run, vbo, &ib->rhdr, bytes, &in->nb); + err = ntfs_read_bh_ra(sbi, run, vbo, &ib->rhdr, bytes, &in->nb, ra); up_read(lock); if (err == -E_NTFS_FIXUP) goto ok; @@ -1100,7 +1099,7 @@ ok: } if (err == -E_NTFS_FIXUP) { - ntfs_write_bh(ni->mi.sbi, &ib->rhdr, &in->nb, 0); + ntfs_write_bh(sbi, &ib->rhdr, &in->nb, 0); err = 0; } @@ -1190,7 +1189,12 @@ int indx_find(struct ntfs_index *indx, struct ntfs_inode *ni, return -EINVAL; } - fnd_push(fnd, node, e); + err = fnd_push(fnd, node, e); + + if (err) { + put_indx_node(node); + return err; + } } *entry = e; @@ -1442,8 +1446,8 @@ static int indx_create_allocate(struct ntfs_index *indx, struct ntfs_inode *ni, run_init(&run); - err = attr_allocate_clusters(sbi, &run, 0, 0, len, NULL, ALLOCATE_DEF, - &alen, 0, NULL, NULL); + err = attr_allocate_clusters(sbi, &run, NULL, 0, 0, len, NULL, + ALLOCATE_DEF, &alen, 0, NULL, NULL); if (err) goto out; @@ -1527,8 +1531,7 @@ static int indx_add_allocate(struct ntfs_index *indx, struct ntfs_inode *ni, /* Increase bitmap. */ err = attr_set_size(ni, ATTR_BITMAP, in->name, in->name_len, &indx->bitmap_run, - ntfs3_bitmap_size(bit + 1), NULL, true, - NULL); + ntfs3_bitmap_size(bit + 1), NULL, true); if (err) goto out1; } @@ -1549,8 +1552,7 @@ static int indx_add_allocate(struct ntfs_index *indx, struct ntfs_inode *ni, /* Increase allocation. */ err = attr_set_size(ni, ATTR_ALLOC, in->name, in->name_len, - &indx->alloc_run, data_size, &data_size, true, - NULL); + &indx->alloc_run, data_size, &data_size, true); if (err) { if (bmp) goto out2; @@ -1568,7 +1570,7 @@ out: out2: /* Ops. No space? */ attr_set_size(ni, ATTR_BITMAP, in->name, in->name_len, - &indx->bitmap_run, bmp_size, &bmp_size_v, false, NULL); + &indx->bitmap_run, bmp_size, &bmp_size_v, false); out1: return err; @@ -1998,6 +2000,7 @@ int indx_insert_entry(struct ntfs_index *indx, struct ntfs_inode *ni, fnd->level - 1, fnd); } + indx->version += 1; out: fnd_put(fnd_a); out1: @@ -2101,7 +2104,7 @@ static int indx_shrink(struct ntfs_index *indx, struct ntfs_inode *ni, new_data = (u64)bit << indx->index_bits; err = attr_set_size(ni, ATTR_ALLOC, in->name, in->name_len, - &indx->alloc_run, new_data, &new_data, false, NULL); + &indx->alloc_run, new_data, &new_data, false); if (err) return err; @@ -2113,7 +2116,7 @@ static int indx_shrink(struct ntfs_index *indx, struct ntfs_inode *ni, return 0; err = attr_set_size(ni, ATTR_BITMAP, in->name, in->name_len, - &indx->bitmap_run, bpb, &bpb, false, NULL); + &indx->bitmap_run, bpb, &bpb, false); return err; } @@ -2328,6 +2331,7 @@ int indx_delete_entry(struct ntfs_index *indx, struct ntfs_inode *ni, hdr = &root->ihdr; e = fnd->root_de; n = NULL; + ib = NULL; } e_size = le16_to_cpu(e->size); @@ -2350,7 +2354,7 @@ int indx_delete_entry(struct ntfs_index *indx, struct ntfs_inode *ni, * Check to see if removing that entry made * the leaf empty. */ - if (ib_is_leaf(ib) && ib_is_empty(ib)) { + if (ib && ib_is_leaf(ib) && ib_is_empty(ib)) { fnd_pop(fnd); fnd_push(fnd2, n, e); } @@ -2598,7 +2602,7 @@ int indx_delete_entry(struct ntfs_index *indx, struct ntfs_inode *ni, in = &s_index_names[indx->type]; err = attr_set_size(ni, ATTR_ALLOC, in->name, in->name_len, - &indx->alloc_run, 0, NULL, false, NULL); + &indx->alloc_run, 0, NULL, false); if (in->name == I30_NAME) i_size_write(&ni->vfs_inode, 0); @@ -2607,7 +2611,7 @@ int indx_delete_entry(struct ntfs_index *indx, struct ntfs_inode *ni, run_close(&indx->alloc_run); err = attr_set_size(ni, ATTR_BITMAP, in->name, in->name_len, - &indx->bitmap_run, 0, NULL, false, NULL); + &indx->bitmap_run, 0, NULL, false); err = ni_remove_attr(ni, ATTR_BITMAP, in->name, in->name_len, false, NULL); run_close(&indx->bitmap_run); @@ -2645,6 +2649,7 @@ int indx_delete_entry(struct ntfs_index *indx, struct ntfs_inode *ni, mi->dirty = true; } + indx->version += 1; out: fnd_put(fnd2); out1: diff --git a/fs/ntfs3/inode.c b/fs/ntfs3/inode.c index edfb973e4e82..6e65066ebcc1 100644 --- a/fs/ntfs3/inode.c +++ b/fs/ntfs3/inode.c @@ -12,6 +12,7 @@ #include #include #include +#include #include "debug.h" #include "ntfs.h" @@ -39,7 +40,7 @@ static struct inode *ntfs_read_mft(struct inode *inode, u32 rp_fa = 0, asize, t32; u16 roff, rsize, names = 0, links = 0; const struct ATTR_FILE_NAME *fname = NULL; - const struct INDEX_ROOT *root; + const struct INDEX_ROOT *root = NULL; struct REPARSE_DATA_BUFFER rp; // 0x18 bytes u64 t64; struct MFT_REC *rec; @@ -166,9 +167,7 @@ next_attr: std5 = Add2Ptr(attr, roff); -#ifdef STATX_BTIME nt2kernel(std5->cr_time, &ni->i_crtime); -#endif nt2kernel(std5->a_time, &ts); inode_set_atime_to_ts(inode, ts); nt2kernel(std5->c_time, &ts); @@ -555,194 +554,148 @@ struct inode *ntfs_iget5(struct super_block *sb, const struct MFT_REF *ref, return inode; } -enum get_block_ctx { - GET_BLOCK_GENERAL = 0, - GET_BLOCK_WRITE_BEGIN = 1, - GET_BLOCK_DIRECT_IO_R = 2, - GET_BLOCK_DIRECT_IO_W = 3, - GET_BLOCK_BMAP = 4, -}; - -static noinline int ntfs_get_block_vbo(struct inode *inode, u64 vbo, - struct buffer_head *bh, int create, - enum get_block_ctx ctx) -{ - struct super_block *sb = inode->i_sb; - struct ntfs_sb_info *sbi = sb->s_fs_info; - struct ntfs_inode *ni = ntfs_i(inode); - struct folio *folio = bh->b_folio; - u8 cluster_bits = sbi->cluster_bits; - u32 block_size = sb->s_blocksize; - u64 bytes, lbo, valid; - u32 off; - int err; - CLST vcn, lcn, len; - bool new; - - /* Clear previous state. */ - clear_buffer_new(bh); - clear_buffer_uptodate(bh); - - if (is_resident(ni)) { - bh->b_blocknr = RESIDENT_LCN; - bh->b_size = block_size; - if (!folio) { - /* direct io (read) or bmap call */ - err = 0; - } else { - ni_lock(ni); - err = attr_data_read_resident(ni, folio); - ni_unlock(ni); - - if (!err) - set_buffer_uptodate(bh); - } - return err; - } - - vcn = vbo >> cluster_bits; - off = vbo & sbi->cluster_mask; - new = false; - - err = attr_data_get_block(ni, vcn, 1, &lcn, &len, create ? &new : NULL, - create && sbi->cluster_size > PAGE_SIZE); - if (err) - goto out; - - if (!len) - return 0; - - bytes = ((u64)len << cluster_bits) - off; - - if (lcn >= sbi->used.bitmap.nbits) { - /* This case includes resident/compressed/sparse. */ - if (!create) { - if (bh->b_size > bytes) - bh->b_size = bytes; - return 0; - } - WARN_ON(1); - } - - if (new) - set_buffer_new(bh); - - lbo = ((u64)lcn << cluster_bits) + off; - - set_buffer_mapped(bh); - bh->b_bdev = sb->s_bdev; - bh->b_blocknr = lbo >> sb->s_blocksize_bits; - - valid = ni->i_valid; - - if (ctx == GET_BLOCK_DIRECT_IO_W) { - /* ntfs_direct_IO will update ni->i_valid. */ - if (vbo >= valid) - set_buffer_new(bh); - } else if (create) { - /* Normal write. */ - if (bytes > bh->b_size) - bytes = bh->b_size; - - if (vbo >= valid) - set_buffer_new(bh); - - if (vbo + bytes > valid) { - ni->i_valid = vbo + bytes; - mark_inode_dirty(inode); - } - } else if (vbo >= valid) { - /* Read out of valid data. */ - clear_buffer_mapped(bh); - } else if (vbo + bytes <= valid) { - /* Normal read. */ - } else if (vbo + block_size <= valid) { - /* Normal short read. */ - bytes = block_size; - } else { - /* - * Read across valid size: vbo < valid && valid < vbo + block_size - */ - bytes = block_size; - - if (folio) { - u32 voff = valid - vbo; - - bh->b_size = block_size; - off = vbo & (PAGE_SIZE - 1); - folio_set_bh(bh, folio, off); - - if (bh_read(bh, 0) < 0) { - err = -EIO; - goto out; - } - folio_zero_segment(folio, off + voff, off + block_size); - } - } - - if (bh->b_size > bytes) - bh->b_size = bytes; - -#ifndef __LP64__ - if (ctx == GET_BLOCK_DIRECT_IO_W || ctx == GET_BLOCK_DIRECT_IO_R) { - static_assert(sizeof(size_t) < sizeof(loff_t)); - if (bytes > 0x40000000u) - bh->b_size = 0x40000000u; - } -#endif - - return 0; - -out: - return err; -} - -int ntfs_get_block(struct inode *inode, sector_t vbn, - struct buffer_head *bh_result, int create) -{ - return ntfs_get_block_vbo(inode, (u64)vbn << inode->i_blkbits, - bh_result, create, GET_BLOCK_GENERAL); -} - -static int ntfs_get_block_bmap(struct inode *inode, sector_t vsn, - struct buffer_head *bh_result, int create) -{ - return ntfs_get_block_vbo(inode, - (u64)vsn << inode->i_sb->s_blocksize_bits, - bh_result, create, GET_BLOCK_BMAP); -} - static sector_t ntfs_bmap(struct address_space *mapping, sector_t block) { - return generic_block_bmap(mapping, block, ntfs_get_block_bmap); + struct inode *inode = mapping->host; + struct ntfs_inode *ni = ntfs_i(inode); + + /* + * We can get here for an inline file via the FIBMAP ioctl + */ + if (is_resident(ni)) + return 0; + + if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) && + !run_is_empty(&ni->file.run_da)) { + /* + * With delalloc data we want to sync the file so + * that we can make sure we allocate blocks for file and data + * is in place for the user to see it + */ + ni_allocate_da_blocks(ni); + } + + return iomap_bmap(mapping, block, &ntfs_iomap_ops); } +static void ntfs_iomap_read_end_io(struct bio *bio) +{ + int error = blk_status_to_errno(bio->bi_status); + struct folio_iter fi; + + bio_for_each_folio_all(fi, bio) { + struct folio *folio = fi.folio; + struct inode *inode = folio->mapping->host; + struct ntfs_inode *ni = ntfs_i(inode); + u64 valid = ni->i_valid; + u32 f_size = folio_size(folio); + loff_t f_pos = folio_pos(folio); + + + if (valid < f_pos + f_size) { + u32 z_from = valid <= f_pos ? + 0 : + offset_in_folio(folio, valid); + /* The only thing ntfs_iomap_read_end_io used for. */ + folio_zero_segment(folio, z_from, f_size); + } + + iomap_finish_folio_read(folio, fi.offset, fi.length, error); + } + bio_put(bio); +} + +/* + * Copied from iomap/bio.c. + */ +static int ntfs_iomap_bio_read_folio_range(const struct iomap_iter *iter, + struct iomap_read_folio_ctx *ctx, + size_t plen) +{ + struct folio *folio = ctx->cur_folio; + const struct iomap *iomap = &iter->iomap; + loff_t pos = iter->pos; + size_t poff = offset_in_folio(folio, pos); + loff_t length = iomap_length(iter); + sector_t sector; + struct bio *bio = ctx->read_ctx; + + sector = iomap_sector(iomap, pos); + if (!bio || bio_end_sector(bio) != sector || + !bio_add_folio(bio, folio, plen, poff)) { + gfp_t gfp = mapping_gfp_constraint(folio->mapping, GFP_KERNEL); + gfp_t orig_gfp = gfp; + unsigned int nr_vecs = DIV_ROUND_UP(length, PAGE_SIZE); + + if (bio) + submit_bio(bio); + + if (ctx->rac) /* same as readahead_gfp_mask */ + gfp |= __GFP_NORETRY | __GFP_NOWARN; + bio = bio_alloc(iomap->bdev, bio_max_segs(nr_vecs), REQ_OP_READ, + gfp); + /* + * If the bio_alloc fails, try it again for a single page to + * avoid having to deal with partial page reads. This emulates + * what do_mpage_read_folio does. + */ + if (!bio) + bio = bio_alloc(iomap->bdev, 1, REQ_OP_READ, orig_gfp); + if (ctx->rac) + bio->bi_opf |= REQ_RAHEAD; + bio->bi_iter.bi_sector = sector; + bio->bi_end_io = ntfs_iomap_read_end_io; + bio_add_folio_nofail(bio, folio, plen, poff); + ctx->read_ctx = bio; + } + return 0; +} + +static void ntfs_iomap_bio_submit_read(struct iomap_read_folio_ctx *ctx) +{ + struct bio *bio = ctx->read_ctx; + + if (bio) + submit_bio(bio); +} + +static const struct iomap_read_ops ntfs_iomap_bio_read_ops = { + .read_folio_range = ntfs_iomap_bio_read_folio_range, + .submit_read = ntfs_iomap_bio_submit_read, +}; + static int ntfs_read_folio(struct file *file, struct folio *folio) { int err; struct address_space *mapping = folio->mapping; struct inode *inode = mapping->host; struct ntfs_inode *ni = ntfs_i(inode); + loff_t vbo = folio_pos(folio); + struct iomap_read_folio_ctx ctx = { + .cur_folio = folio, + .ops = &ntfs_iomap_bio_read_ops, + }; - if (is_resident(ni)) { - ni_lock(ni); - err = attr_data_read_resident(ni, folio); - ni_unlock(ni); - if (err != E_NTFS_NONRESIDENT) { - folio_unlock(folio); - return err; - } + if (unlikely(is_bad_ni(ni))) { + folio_unlock(folio); + return -EIO; + } + + if (ni->i_valid <= vbo) { + folio_zero_range(folio, 0, folio_size(folio)); + folio_mark_uptodate(folio); + folio_unlock(folio); + return 0; } if (is_compressed(ni)) { - ni_lock(ni); - err = ni_readpage_cmpr(ni, folio); - ni_unlock(ni); + /* ni_lock is taken inside ni_read_folio_cmpr after page locks */ + err = ni_read_folio_cmpr(ni, folio); return err; } - /* Normal + sparse files. */ - return mpage_read_folio(folio, ntfs_get_block); + iomap_read_folio(&ntfs_iomap_ops, &ctx, NULL); + return 0; } static void ntfs_readahead(struct readahead_control *rac) @@ -750,8 +703,10 @@ static void ntfs_readahead(struct readahead_control *rac) struct address_space *mapping = rac->mapping; struct inode *inode = mapping->host; struct ntfs_inode *ni = ntfs_i(inode); - u64 valid; - loff_t pos; + struct iomap_read_folio_ctx ctx = { + .ops = &ntfs_iomap_bio_read_ops, + .rac = rac, + }; if (is_resident(ni)) { /* No readahead for resident. */ @@ -763,80 +718,7 @@ static void ntfs_readahead(struct readahead_control *rac) return; } - valid = ni->i_valid; - pos = readahead_pos(rac); - - if (valid < i_size_read(inode) && pos <= valid && - valid < pos + readahead_length(rac)) { - /* Range cross 'valid'. Read it page by page. */ - return; - } - - mpage_readahead(rac, ntfs_get_block); -} - -static int ntfs_get_block_direct_IO_R(struct inode *inode, sector_t iblock, - struct buffer_head *bh_result, int create) -{ - return ntfs_get_block_vbo(inode, (u64)iblock << inode->i_blkbits, - bh_result, create, GET_BLOCK_DIRECT_IO_R); -} - -static int ntfs_get_block_direct_IO_W(struct inode *inode, sector_t iblock, - struct buffer_head *bh_result, int create) -{ - return ntfs_get_block_vbo(inode, (u64)iblock << inode->i_blkbits, - bh_result, create, GET_BLOCK_DIRECT_IO_W); -} - -static ssize_t ntfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter) -{ - struct file *file = iocb->ki_filp; - struct address_space *mapping = file->f_mapping; - struct inode *inode = mapping->host; - struct ntfs_inode *ni = ntfs_i(inode); - loff_t vbo = iocb->ki_pos; - loff_t end; - int wr = iov_iter_rw(iter) & WRITE; - size_t iter_count = iov_iter_count(iter); - loff_t valid; - ssize_t ret; - - if (is_resident(ni)) { - /* Switch to buffered write. */ - ret = 0; - goto out; - } - if (is_compressed(ni)) { - ret = 0; - goto out; - } - - ret = blockdev_direct_IO(iocb, inode, iter, - wr ? ntfs_get_block_direct_IO_W : - ntfs_get_block_direct_IO_R); - - if (ret > 0) - end = vbo + ret; - else if (wr && ret == -EIOCBQUEUED) - end = vbo + iter_count; - else - goto out; - - valid = ni->i_valid; - if (wr) { - if (end > valid && !S_ISBLK(inode->i_mode)) { - ni->i_valid = end; - mark_inode_dirty(inode); - } - } else if (vbo < valid && valid < end) { - /* Fix page. */ - iov_iter_revert(iter, end - valid); - iov_iter_zero(end - valid, iter); - } - -out: - return ret; + iomap_readahead(&ntfs_iomap_ops, &ctx, NULL); } int ntfs_set_size(struct inode *inode, u64 new_size) @@ -849,29 +731,311 @@ int ntfs_set_size(struct inode *inode, u64 new_size) /* Check for maximum file size. */ if (is_sparsed(ni) || is_compressed(ni)) { if (new_size > sbi->maxbytes_sparse) { - err = -EFBIG; - goto out; + return -EFBIG; } } else if (new_size > sbi->maxbytes) { - err = -EFBIG; - goto out; + return -EFBIG; } ni_lock(ni); down_write(&ni->file.run_lock); err = attr_set_size(ni, ATTR_DATA, NULL, 0, &ni->file.run, new_size, - &ni->i_valid, true, NULL); + &ni->i_valid, true); + + if (!err) { + i_size_write(inode, new_size); + mark_inode_dirty(inode); + } up_write(&ni->file.run_lock); ni_unlock(ni); - mark_inode_dirty(inode); - -out: return err; } +/* + * Special value to detect ntfs_writeback_range call + */ +#define WB_NO_DA (struct iomap *)1 +/* + * Function to get mapping vbo -> lbo. + * used with: + * - iomap_zero_range + * - iomap_truncate_page + * - iomap_dio_rw + * - iomap_file_buffered_write + * - iomap_bmap + * - iomap_fiemap + * - iomap_bio_read_folio + * - iomap_bio_readahead + */ +static int ntfs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, + unsigned int flags, struct iomap *iomap, + struct iomap *srcmap) +{ + struct ntfs_inode *ni = ntfs_i(inode); + struct ntfs_sb_info *sbi = ni->mi.sbi; + u8 cluster_bits = sbi->cluster_bits; + CLST vcn = offset >> cluster_bits; + u32 off = offset & sbi->cluster_mask; + bool rw = flags & IOMAP_WRITE; + loff_t endbyte = offset + length; + void *res = NULL; + int err; + CLST lcn, clen, clen_max = 1; + bool new_clst = false; + bool no_da; + bool zero = false; + if (unlikely(ntfs3_forced_shutdown(sbi->sb))) + return -EIO; + + if (flags & IOMAP_REPORT) { + if (offset > ntfs_get_maxbytes(ni)) { + /* called from fiemap/bmap. */ + return -EINVAL; + } + + if (offset >= inode->i_size) { + /* special code for report. */ + return -ENOENT; + } + } + + if (IOMAP_ZERO == flags && (endbyte & sbi->cluster_mask)) { + rw = true; + } else if (rw) { + clen_max = bytes_to_cluster(sbi, endbyte) - vcn; + } + + /* + * Force to allocate clusters if directIO(write) or writeback_range. + * NOTE: attr_data_get_block allocates clusters only for sparse file. + * Normal file allocates clusters in attr_set_size. + */ + no_da = flags == (IOMAP_DIRECT | IOMAP_WRITE) || srcmap == WB_NO_DA; + + err = attr_data_get_block(ni, vcn, clen_max, &lcn, &clen, + rw ? &new_clst : NULL, zero, &res, no_da); + + if (err) { + return err; + } + + if (lcn == EOF_LCN) { + /* request out of file. */ + if (flags & IOMAP_REPORT) { + /* special code for report. */ + return -ENOENT; + } + + if (rw) { + /* should never be here. */ + return -EINVAL; + } + lcn = SPARSE_LCN; + } + + iomap->flags = new_clst ? IOMAP_F_NEW : 0; + + if (lcn == RESIDENT_LCN) { + if (offset >= clen) { + kfree(res); + if (flags & IOMAP_REPORT) { + /* special code for report. */ + return -ENOENT; + } + return -EFAULT; + } + + iomap->private = iomap->inline_data = res; + iomap->type = IOMAP_INLINE; + iomap->offset = 0; + iomap->length = clen; /* resident size in bytes. */ + return 0; + } + + if (!clen) { + /* broken file? */ + return -EINVAL; + } + + iomap->bdev = inode->i_sb->s_bdev; + iomap->offset = offset; + iomap->length = ((loff_t)clen << cluster_bits) - off; + + if (lcn == COMPRESSED_LCN) { + /* should never be here. */ + return -EOPNOTSUPP; + } + + if (lcn == DELALLOC_LCN) { + iomap->type = IOMAP_DELALLOC; + iomap->addr = IOMAP_NULL_ADDR; + } else { + + /* Translate clusters into bytes. */ + iomap->addr = ((loff_t)lcn << cluster_bits) + off; + if (length && iomap->length > length) + iomap->length = length; + else + endbyte = offset + iomap->length; + + if (lcn == SPARSE_LCN) { + iomap->addr = IOMAP_NULL_ADDR; + iomap->type = IOMAP_HOLE; + // if (IOMAP_ZERO == flags && !off) { + // iomap->length = (endbyte - offset) & + // sbi->cluster_mask_inv; + // } + } else if (endbyte <= ni->i_valid) { + iomap->type = IOMAP_MAPPED; + } else if (offset < ni->i_valid) { + iomap->type = IOMAP_MAPPED; + if (flags & IOMAP_REPORT) + iomap->length = ni->i_valid - offset; + } else if (rw || (flags & IOMAP_ZERO)) { + iomap->type = IOMAP_MAPPED; + } else { + iomap->type = IOMAP_UNWRITTEN; + } + } + + if ((flags & IOMAP_ZERO) && + (iomap->type == IOMAP_MAPPED || iomap->type == IOMAP_DELALLOC)) { + /* Avoid too large requests. */ + u32 tail; + u32 off_a = offset & (PAGE_SIZE - 1); + if (off_a) + tail = PAGE_SIZE - off_a; + else + tail = PAGE_SIZE; + + if (iomap->length > tail) + iomap->length = tail; + } + + return 0; +} + +static int ntfs_iomap_end(struct inode *inode, loff_t pos, loff_t length, + ssize_t written, unsigned int flags, + struct iomap *iomap) +{ + int err = 0; + struct ntfs_inode *ni = ntfs_i(inode); + loff_t endbyte = pos + written; + + if ((flags & IOMAP_WRITE) || (flags & IOMAP_ZERO)) { + if (iomap->type == IOMAP_INLINE) { + u32 data_size; + struct ATTRIB *attr; + struct mft_inode *mi; + + attr = ni_find_attr(ni, NULL, NULL, ATTR_DATA, NULL, 0, + NULL, &mi); + if (!attr || attr->non_res) { + err = -EINVAL; + goto out; + } + + data_size = le32_to_cpu(attr->res.data_size); + if (!(pos < data_size && endbyte <= data_size)) { + err = -EINVAL; + goto out; + } + + /* Update resident data. */ + memcpy(resident_data(attr) + pos, + iomap_inline_data(iomap, pos), written); + mi->dirty = true; + ni->i_valid = data_size; + } else if (ni->i_valid < endbyte) { + ni->i_valid = endbyte; + mark_inode_dirty(inode); + } + } + + if ((flags & IOMAP_ZERO) && + (iomap->type == IOMAP_MAPPED || iomap->type == IOMAP_DELALLOC)) { + /* Pair for code in ntfs_iomap_begin. */ + balance_dirty_pages_ratelimited(inode->i_mapping); + cond_resched(); + } + +out: + if (iomap->type == IOMAP_INLINE) { + kfree(iomap->private); + iomap->private = NULL; + } + + return err; +} + +/* + * write_begin + put_folio + write_end. + * iomap_zero_range + * iomap_truncate_page + * iomap_file_buffered_write + */ +static void ntfs_iomap_put_folio(struct inode *inode, loff_t pos, + unsigned int len, struct folio *folio) +{ + struct ntfs_inode *ni = ntfs_i(inode); + loff_t end = pos + len; + u32 f_size = folio_size(folio); + loff_t f_pos = folio_pos(folio); + loff_t f_end = f_pos + f_size; + + if (ni->i_valid <= end && end < f_end) { + /* zero range [end - f_end). */ + /* The only thing ntfs_iomap_put_folio used for. */ + folio_zero_segment(folio, offset_in_folio(folio, end), f_size); + } + folio_unlock(folio); + folio_put(folio); +} + +/* + * iomap_writeback_ops::writeback_range + */ +static ssize_t ntfs_writeback_range(struct iomap_writepage_ctx *wpc, + struct folio *folio, u64 offset, + unsigned int len, u64 end_pos) +{ + struct iomap *iomap = &wpc->iomap; + /* Check iomap position. */ + if (iomap->offset + iomap->length <= offset || offset < iomap->offset) { + int err; + struct inode *inode = wpc->inode; + struct ntfs_inode *ni = ntfs_i(inode); + struct ntfs_sb_info *sbi = ntfs_sb(inode->i_sb); + loff_t i_size_up = ntfs_up_cluster(sbi, inode->i_size); + loff_t len_max = i_size_up - offset; + + err = ni->file.run_da.count ? ni_allocate_da_blocks(ni) : 0; + + if (!err) { + /* Use local special value 'WB_NO_DA' to disable delalloc. */ + err = ntfs_iomap_begin(inode, offset, len_max, + IOMAP_WRITE, iomap, WB_NO_DA); + } + + if (err) { + ntfs_set_state(sbi, NTFS_DIRTY_DIRTY); + return err; + } + } + + return iomap_add_to_ioend(wpc, folio, offset, end_pos, len); +} + + +static const struct iomap_writeback_ops ntfs_writeback_ops = { + .writeback_range = ntfs_writeback_range, + .writeback_submit = iomap_ioend_writeback_submit, +}; + static int ntfs_resident_writepage(struct folio *folio, struct writeback_control *wbc) { @@ -899,40 +1063,15 @@ static int ntfs_resident_writepage(struct folio *folio, static int ntfs_writepages(struct address_space *mapping, struct writeback_control *wbc) -{ - struct inode *inode = mapping->host; - - /* Avoid any operation if inode is bad. */ - if (unlikely(is_bad_ni(ntfs_i(inode)))) - return -EINVAL; - - if (unlikely(ntfs3_forced_shutdown(inode->i_sb))) - return -EIO; - - if (is_resident(ntfs_i(inode))) { - struct folio *folio = NULL; - int error; - - while ((folio = writeback_iter(mapping, wbc, folio, &error))) - error = ntfs_resident_writepage(folio, wbc); - return error; - } - return mpage_writepages(mapping, wbc, ntfs_get_block); -} - -static int ntfs_get_block_write_begin(struct inode *inode, sector_t vbn, - struct buffer_head *bh_result, int create) -{ - return ntfs_get_block_vbo(inode, (u64)vbn << inode->i_blkbits, - bh_result, create, GET_BLOCK_WRITE_BEGIN); -} - -int ntfs_write_begin(const struct kiocb *iocb, struct address_space *mapping, - loff_t pos, u32 len, struct folio **foliop, void **fsdata) { int err; struct inode *inode = mapping->host; struct ntfs_inode *ni = ntfs_i(inode); + struct iomap_writepage_ctx wpc = { + .inode = mapping->host, + .wbc = wbc, + .ops = &ntfs_writeback_ops, + }; /* Avoid any operation if inode is bad. */ if (unlikely(is_bad_ni(ni))) @@ -942,100 +1081,15 @@ int ntfs_write_begin(const struct kiocb *iocb, struct address_space *mapping, return -EIO; if (is_resident(ni)) { - struct folio *folio = __filemap_get_folio( - mapping, pos >> PAGE_SHIFT, FGP_WRITEBEGIN, - mapping_gfp_mask(mapping)); + struct folio *folio = NULL; - if (IS_ERR(folio)) { - err = PTR_ERR(folio); - goto out; - } + while ((folio = writeback_iter(mapping, wbc, folio, &err))) + err = ntfs_resident_writepage(folio, wbc); - ni_lock(ni); - err = attr_data_read_resident(ni, folio); - ni_unlock(ni); - - if (!err) { - *foliop = folio; - goto out; - } - folio_unlock(folio); - folio_put(folio); - - if (err != E_NTFS_NONRESIDENT) - goto out; + return err; } - err = block_write_begin(mapping, pos, len, foliop, - ntfs_get_block_write_begin); - -out: - return err; -} - -/* - * ntfs_write_end - Address_space_operations::write_end. - */ -int ntfs_write_end(const struct kiocb *iocb, struct address_space *mapping, - loff_t pos, u32 len, u32 copied, struct folio *folio, - void *fsdata) -{ - struct inode *inode = mapping->host; - struct ntfs_inode *ni = ntfs_i(inode); - u64 valid = ni->i_valid; - bool dirty = false; - int err; - - if (is_resident(ni)) { - ni_lock(ni); - err = attr_data_write_resident(ni, folio); - ni_unlock(ni); - if (!err) { - struct buffer_head *head = folio_buffers(folio); - dirty = true; - /* Clear any buffers in folio. */ - if (head) { - struct buffer_head *bh = head; - - do { - clear_buffer_dirty(bh); - clear_buffer_mapped(bh); - set_buffer_uptodate(bh); - } while (head != (bh = bh->b_this_page)); - } - folio_mark_uptodate(folio); - err = copied; - } - folio_unlock(folio); - folio_put(folio); - } else { - err = generic_write_end(iocb, mapping, pos, len, copied, folio, - fsdata); - } - - if (err >= 0) { - if (!(ni->std_fa & FILE_ATTRIBUTE_ARCHIVE)) { - inode_set_mtime_to_ts(inode, - inode_set_ctime_current(inode)); - ni->std_fa |= FILE_ATTRIBUTE_ARCHIVE; - dirty = true; - } - - if (valid != ni->i_valid) { - /* ni->i_valid is changed in ntfs_get_block_vbo. */ - dirty = true; - } - - if (pos + err > inode->i_size) { - i_size_write(inode, pos + err); - dirty = true; - } - - if (dirty) - mark_inode_dirty(inode); - } - - return err; + return iomap_writepages(&wpc); } int ntfs3_write_inode(struct inode *inode, struct writeback_control *wbc) @@ -1050,6 +1104,7 @@ int ntfs_sync_inode(struct inode *inode) /* * Helper function to read file. + * Used to read $AttrDef and $UpCase */ int inode_read_data(struct inode *inode, void *data, size_t bytes) { @@ -1539,9 +1594,10 @@ int ntfs_create_inode(struct mnt_idmap *idmap, struct inode *dir, attr->nres.alloc_size = cpu_to_le64(ntfs_up_cluster(sbi, nsize)); - err = attr_allocate_clusters(sbi, &ni->file.run, 0, 0, - clst, NULL, ALLOCATE_DEF, - &alen, 0, NULL, NULL); + err = attr_allocate_clusters(sbi, &ni->file.run, NULL, + 0, 0, clst, NULL, + ALLOCATE_DEF, &alen, 0, + NULL, NULL); if (err) goto out5; @@ -1682,7 +1738,7 @@ out6: /* Delete ATTR_EA, if non-resident. */ struct runs_tree run; run_init(&run); - attr_set_size(ni, ATTR_EA, NULL, 0, &run, 0, NULL, false, NULL); + attr_set_size(ni, ATTR_EA, NULL, 0, &run, 0, NULL, false); run_close(&run); } @@ -2094,18 +2150,26 @@ const struct address_space_operations ntfs_aops = { .read_folio = ntfs_read_folio, .readahead = ntfs_readahead, .writepages = ntfs_writepages, - .write_begin = ntfs_write_begin, - .write_end = ntfs_write_end, - .direct_IO = ntfs_direct_IO, .bmap = ntfs_bmap, - .dirty_folio = block_dirty_folio, - .migrate_folio = buffer_migrate_folio, - .invalidate_folio = block_invalidate_folio, + .dirty_folio = iomap_dirty_folio, + .migrate_folio = filemap_migrate_folio, + .release_folio = iomap_release_folio, + .invalidate_folio = iomap_invalidate_folio, }; const struct address_space_operations ntfs_aops_cmpr = { .read_folio = ntfs_read_folio, - .dirty_folio = block_dirty_folio, - .direct_IO = ntfs_direct_IO, + .dirty_folio = iomap_dirty_folio, + .release_folio = iomap_release_folio, + .invalidate_folio = iomap_invalidate_folio, +}; + +const struct iomap_ops ntfs_iomap_ops = { + .iomap_begin = ntfs_iomap_begin, + .iomap_end = ntfs_iomap_end, +}; + +const struct iomap_write_ops ntfs_iomap_folio_ops = { + .put_folio = ntfs_iomap_put_folio, }; // clang-format on diff --git a/fs/ntfs3/ntfs.h b/fs/ntfs3/ntfs.h index 552b97905813..892f13e65d42 100644 --- a/fs/ntfs3/ntfs.h +++ b/fs/ntfs3/ntfs.h @@ -77,10 +77,14 @@ static_assert(sizeof(size_t) == 8); typedef u32 CLST; #endif +/* On-disk sparsed cluster is marked as -1. */ #define SPARSE_LCN64 ((u64)-1) #define SPARSE_LCN ((CLST)-1) +/* Below is virtual (not on-disk) values. */ #define RESIDENT_LCN ((CLST)-2) #define COMPRESSED_LCN ((CLST)-3) +#define EOF_LCN ((CLST)-4) +#define DELALLOC_LCN ((CLST)-5) enum RECORD_NUM { MFT_REC_MFT = 0, diff --git a/fs/ntfs3/ntfs_fs.h b/fs/ntfs3/ntfs_fs.h index f18349689458..921b526eb0f4 100644 --- a/fs/ntfs3/ntfs_fs.h +++ b/fs/ntfs3/ntfs_fs.h @@ -109,6 +109,7 @@ struct ntfs_mount_options { unsigned force : 1; /* RW mount dirty volume. */ unsigned prealloc : 1; /* Preallocate space when file is growing. */ unsigned nocase : 1; /* case insensitive. */ + unsigned delalloc : 1; /* delay allocation. */ }; /* Special value to unpack and deallocate. */ @@ -133,7 +134,8 @@ struct ntfs_buffers { enum ALLOCATE_OPT { ALLOCATE_DEF = 0, // Allocate all clusters. ALLOCATE_MFT = 1, // Allocate for MFT. - ALLOCATE_ZERO = 2, // Zeroout new allocated clusters + ALLOCATE_ZERO = 2, // Zeroout new allocated clusters. + ALLOCATE_ONE_FR = 4, // Allocate one fragment only. }; enum bitmap_mutex_classes { @@ -192,6 +194,7 @@ struct ntfs_index { struct runs_tree alloc_run; /* read/write access to 'bitmap_run'/'alloc_run' while ntfs_readdir */ struct rw_semaphore run_lock; + size_t version; /* increment each change */ /*TODO: Remove 'cmp'. */ NTFS_CMP_FUNC cmp; @@ -213,7 +216,7 @@ struct ntfs_sb_info { u32 discard_granularity; u64 discard_granularity_mask_inv; // ~(discard_granularity_mask_inv-1) - u32 bdev_blocksize_mask; // bdev_logical_block_size(bdev) - 1; + u32 bdev_blocksize; // bdev_logical_block_size(bdev) u32 cluster_size; // bytes per cluster u32 cluster_mask; // == cluster_size - 1 @@ -272,6 +275,12 @@ struct ntfs_sb_info { struct { struct wnd_bitmap bitmap; // $Bitmap::Data CLST next_free_lcn; + /* Total sum of delay allocated clusters in all files. */ +#ifdef CONFIG_NTFS3_64BIT_CLUSTER + atomic64_t da; +#else + atomic_t da; +#endif } used; struct { @@ -379,7 +388,7 @@ struct ntfs_inode { */ u8 mi_loaded; - /* + /* * Use this field to avoid any write(s). * If inode is bad during initialization - use make_bad_inode * If inode is bad during operations - use this field @@ -390,7 +399,14 @@ struct ntfs_inode { struct ntfs_index dir; struct { struct rw_semaphore run_lock; + /* Unpacked runs from just one record. */ struct runs_tree run; + /* + * Pairs [vcn, len] for all delay allocated clusters. + * Normal file always contains delayed clusters in one fragment. + * TODO: use 2 CLST per pair instead of 3. + */ + struct runs_tree run_da; #ifdef CONFIG_NTFS3_LZX_XPRESS struct folio *offs_folio; #endif @@ -430,20 +446,32 @@ enum REPARSE_SIGN { /* Functions from attrib.c */ int attr_allocate_clusters(struct ntfs_sb_info *sbi, struct runs_tree *run, - CLST vcn, CLST lcn, CLST len, CLST *pre_alloc, - enum ALLOCATE_OPT opt, CLST *alen, const size_t fr, - CLST *new_lcn, CLST *new_len); + struct runs_tree *run_da, CLST vcn, CLST lcn, + CLST len, CLST *pre_alloc, enum ALLOCATE_OPT opt, + CLST *alen, const size_t fr, CLST *new_lcn, + CLST *new_len); int attr_make_nonresident(struct ntfs_inode *ni, struct ATTRIB *attr, struct ATTR_LIST_ENTRY *le, struct mft_inode *mi, u64 new_size, struct runs_tree *run, struct ATTRIB **ins_attr, struct page *page); -int attr_set_size(struct ntfs_inode *ni, enum ATTR_TYPE type, - const __le16 *name, u8 name_len, struct runs_tree *run, - u64 new_size, const u64 *new_valid, bool keep_prealloc, - struct ATTRIB **ret); +int attr_set_size_ex(struct ntfs_inode *ni, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, struct runs_tree *run, + u64 new_size, const u64 *new_valid, bool keep_prealloc, + struct ATTRIB **ret, bool no_da); +static inline int attr_set_size(struct ntfs_inode *ni, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, + struct runs_tree *run, u64 new_size, + const u64 *new_valid, bool keep_prealloc) +{ + return attr_set_size_ex(ni, type, name, name_len, run, new_size, + new_valid, keep_prealloc, NULL, false); +} int attr_data_get_block(struct ntfs_inode *ni, CLST vcn, CLST clen, CLST *lcn, - CLST *len, bool *new, bool zero); -int attr_data_read_resident(struct ntfs_inode *ni, struct folio *folio); + CLST *len, bool *new, bool zero, void **res, + bool no_da); +int attr_data_get_block_locked(struct ntfs_inode *ni, CLST vcn, CLST clen, + CLST *lcn, CLST *len, bool *new, bool zero, + void **res, bool no_da); int attr_data_write_resident(struct ntfs_inode *ni, struct folio *folio); int attr_load_runs_vcn(struct ntfs_inode *ni, enum ATTR_TYPE type, const __le16 *name, u8 name_len, struct runs_tree *run, @@ -512,6 +540,7 @@ int ntfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, int ntfs_file_open(struct inode *inode, struct file *file); int ntfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, __u64 start, __u64 len); +int ntfs_file_fsync(struct file *file, loff_t start, loff_t end, int datasync); long ntfs_ioctl(struct file *filp, u32 cmd, unsigned long arg); long ntfs_compat_ioctl(struct file *filp, u32 cmd, unsigned long arg); extern const struct inode_operations ntfs_special_inode_operations; @@ -567,9 +596,7 @@ enum REPARSE_SIGN ni_parse_reparse(struct ntfs_inode *ni, struct ATTRIB *attr, struct REPARSE_DATA_BUFFER *buffer); int ni_write_inode(struct inode *inode, int sync, const char *hint); #define _ni_write_inode(i, w) ni_write_inode(i, w, __func__) -int ni_fiemap(struct ntfs_inode *ni, struct fiemap_extent_info *fieinfo, - __u64 vbo, __u64 len); -int ni_readpage_cmpr(struct ntfs_inode *ni, struct folio *folio); +int ni_read_folio_cmpr(struct ntfs_inode *ni, struct folio *folio); int ni_decompress_file(struct ntfs_inode *ni); int ni_read_frame(struct ntfs_inode *ni, u64 frame_vbo, struct page **pages, u32 pages_per_frame, int copy); @@ -590,6 +617,10 @@ int ni_rename(struct ntfs_inode *dir_ni, struct ntfs_inode *new_dir_ni, struct NTFS_DE *new_de); bool ni_is_dirty(struct inode *inode); +loff_t ni_seek_data_or_hole(struct ntfs_inode *ni, loff_t offset, bool data); +int ni_write_parents(struct ntfs_inode *ni, int sync); +int ni_allocate_da_blocks(struct ntfs_inode *ni); +int ni_allocate_da_blocks_locked(struct ntfs_inode *ni); /* Globals from fslog.c */ bool check_index_header(const struct INDEX_HDR *hdr, size_t bytes); @@ -605,13 +636,14 @@ int ntfs_loadlog_and_replay(struct ntfs_inode *ni, struct ntfs_sb_info *sbi); int ntfs_look_for_free_space(struct ntfs_sb_info *sbi, CLST lcn, CLST len, CLST *new_lcn, CLST *new_len, enum ALLOCATE_OPT opt); -bool ntfs_check_for_free_space(struct ntfs_sb_info *sbi, CLST clen, CLST mlen); +bool ntfs_check_free_space(struct ntfs_sb_info *sbi, CLST clen, CLST mlen, + bool da); int ntfs_look_free_mft(struct ntfs_sb_info *sbi, CLST *rno, bool mft, struct ntfs_inode *ni, struct mft_inode **mi); void ntfs_mark_rec_free(struct ntfs_sb_info *sbi, CLST rno, bool is_mft); int ntfs_clear_mft_tail(struct ntfs_sb_info *sbi, size_t from, size_t to); int ntfs_refresh_zone(struct ntfs_sb_info *sbi); -void ntfs_update_mftmirr(struct ntfs_sb_info *sbi, int wait); +void ntfs_update_mftmirr(struct ntfs_sb_info *sbi); void ntfs_bad_inode(struct inode *inode, const char *hint); #define _ntfs_bad_inode(i) ntfs_bad_inode(i, __func__) enum NTFS_DIRTY_FLAGS { @@ -626,11 +658,27 @@ int ntfs_sb_write_run(struct ntfs_sb_info *sbi, const struct runs_tree *run, u64 vbo, const void *buf, size_t bytes, int sync); struct buffer_head *ntfs_bread_run(struct ntfs_sb_info *sbi, const struct runs_tree *run, u64 vbo); -int ntfs_read_run_nb(struct ntfs_sb_info *sbi, const struct runs_tree *run, - u64 vbo, void *buf, u32 bytes, struct ntfs_buffers *nb); -int ntfs_read_bh(struct ntfs_sb_info *sbi, const struct runs_tree *run, u64 vbo, - struct NTFS_RECORD_HEADER *rhdr, u32 bytes, - struct ntfs_buffers *nb); +int ntfs_read_run_nb_ra(struct ntfs_sb_info *sbi, const struct runs_tree *run, + u64 vbo, void *buf, u32 bytes, struct ntfs_buffers *nb, + struct file_ra_state *ra); +static inline int ntfs_read_run_nb(struct ntfs_sb_info *sbi, + const struct runs_tree *run, u64 vbo, + void *buf, u32 bytes, + struct ntfs_buffers *nb) +{ + return ntfs_read_run_nb_ra(sbi, run, vbo, buf, bytes, nb, NULL); +} +int ntfs_read_bh_ra(struct ntfs_sb_info *sbi, const struct runs_tree *run, + u64 vbo, struct NTFS_RECORD_HEADER *rhdr, u32 bytes, + struct ntfs_buffers *nb, struct file_ra_state *ra); +static inline int ntfs_read_bh(struct ntfs_sb_info *sbi, + const struct runs_tree *run, u64 vbo, + struct NTFS_RECORD_HEADER *rhdr, u32 bytes, + struct ntfs_buffers *nb) +{ + return ntfs_read_bh_ra(sbi, run, vbo, rhdr, bytes, nb, NULL); +} + int ntfs_get_bh(struct ntfs_sb_info *sbi, const struct runs_tree *run, u64 vbo, u32 bytes, struct ntfs_buffers *nb); int ntfs_write_bh(struct ntfs_sb_info *sbi, struct NTFS_RECORD_HEADER *rhdr, @@ -696,8 +744,13 @@ int indx_init(struct ntfs_index *indx, struct ntfs_sb_info *sbi, const struct ATTRIB *attr, enum index_mutex_classed type); struct INDEX_ROOT *indx_get_root(struct ntfs_index *indx, struct ntfs_inode *ni, struct ATTRIB **attr, struct mft_inode **mi); -int indx_read(struct ntfs_index *idx, struct ntfs_inode *ni, CLST vbn, - struct indx_node **node); +int indx_read_ra(struct ntfs_index *idx, struct ntfs_inode *ni, CLST vbn, + struct indx_node **node, struct file_ra_state *ra); +static inline int indx_read(struct ntfs_index *idx, struct ntfs_inode *ni, + CLST vbn, struct indx_node **node) +{ + return indx_read_ra(idx, ni, vbn, node, NULL); +} int indx_find(struct ntfs_index *indx, struct ntfs_inode *dir, const struct INDEX_ROOT *root, const void *Key, size_t KeyLen, const void *param, int *diff, struct NTFS_DE **entry, @@ -721,13 +774,6 @@ int indx_update_dup(struct ntfs_inode *ni, struct ntfs_sb_info *sbi, struct inode *ntfs_iget5(struct super_block *sb, const struct MFT_REF *ref, const struct cpu_str *name); int ntfs_set_size(struct inode *inode, u64 new_size); -int ntfs_get_block(struct inode *inode, sector_t vbn, - struct buffer_head *bh_result, int create); -int ntfs_write_begin(const struct kiocb *iocb, struct address_space *mapping, - loff_t pos, u32 len, struct folio **foliop, void **fsdata); -int ntfs_write_end(const struct kiocb *iocb, struct address_space *mapping, - loff_t pos, u32 len, u32 copied, struct folio *folio, - void *fsdata); int ntfs3_write_inode(struct inode *inode, struct writeback_control *wbc); int ntfs_sync_inode(struct inode *inode); int inode_read_data(struct inode *inode, void *data, size_t bytes); @@ -738,6 +784,8 @@ int ntfs_create_inode(struct mnt_idmap *idmap, struct inode *dir, int ntfs_link_inode(struct inode *inode, struct dentry *dentry); int ntfs_unlink_inode(struct inode *dir, const struct dentry *dentry); void ntfs_evict_inode(struct inode *inode); +extern const struct iomap_ops ntfs_iomap_ops; +extern const struct iomap_write_ops ntfs_iomap_folio_ops; extern const struct inode_operations ntfs_link_inode_operations; extern const struct address_space_operations ntfs_aops; extern const struct address_space_operations ntfs_aops_cmpr; @@ -815,7 +863,8 @@ void run_truncate_around(struct runs_tree *run, CLST vcn); bool run_add_entry(struct runs_tree *run, CLST vcn, CLST lcn, CLST len, bool is_mft); bool run_collapse_range(struct runs_tree *run, CLST vcn, CLST len, CLST sub); -bool run_insert_range(struct runs_tree *run, CLST vcn, CLST len); +int run_insert_range(struct runs_tree *run, CLST vcn, CLST len); +int run_insert_range_da(struct runs_tree *run, CLST vcn, CLST len); bool run_get_entry(const struct runs_tree *run, size_t index, CLST *vcn, CLST *lcn, CLST *len); bool run_is_mapped_full(const struct runs_tree *run, CLST svcn, CLST evcn); @@ -835,6 +884,9 @@ int run_unpack_ex(struct runs_tree *run, struct ntfs_sb_info *sbi, CLST ino, #endif int run_get_highest_vcn(CLST vcn, const u8 *run_buf, u64 *highest_vcn); int run_clone(const struct runs_tree *run, struct runs_tree *new_run); +bool run_remove_range(struct runs_tree *run, CLST vcn, CLST len, CLST *done); +CLST run_len(const struct runs_tree *run); +CLST run_get_max_vcn(const struct runs_tree *run); /* Globals from super.c */ void *ntfs_set_shared(void *ptr, u32 bytes); @@ -1011,6 +1063,36 @@ static inline int ntfs3_forced_shutdown(struct super_block *sb) return test_bit(NTFS_FLAGS_SHUTDOWN_BIT, &ntfs_sb(sb)->flags); } +/* Returns total sum of delay allocated clusters in all files. */ +static inline CLST ntfs_get_da(struct ntfs_sb_info *sbi) +{ +#ifdef CONFIG_NTFS3_64BIT_CLUSTER + return atomic64_read(&sbi->used.da); +#else + return atomic_read(&sbi->used.da); +#endif +} + +/* Update total count of delay allocated clusters. */ +static inline void ntfs_add_da(struct ntfs_sb_info *sbi, CLST da) +{ +#ifdef CONFIG_NTFS3_64BIT_CLUSTER + atomic64_add(da, &sbi->used.da); +#else + atomic_add(da, &sbi->used.da); +#endif +} + +/* Update total count of delay allocated clusters. */ +static inline void ntfs_sub_da(struct ntfs_sb_info *sbi, CLST da) +{ +#ifdef CONFIG_NTFS3_64BIT_CLUSTER + atomic64_sub(da, &sbi->used.da); +#else + atomic_sub(da, &sbi->used.da); +#endif +} + /* * ntfs_up_cluster - Align up on cluster boundary. */ @@ -1084,6 +1166,13 @@ static inline int is_resident(struct ntfs_inode *ni) return ni->ni_flags & NI_FLAG_RESIDENT; } +static inline loff_t ntfs_get_maxbytes(struct ntfs_inode *ni) +{ + struct ntfs_sb_info *sbi = ni->mi.sbi; + return is_sparsed(ni) || is_compressed(ni) ? sbi->maxbytes_sparse : + sbi->maxbytes; +} + static inline void le16_sub_cpu(__le16 *var, u16 val) { *var = cpu_to_le16(le16_to_cpu(*var) - val); diff --git a/fs/ntfs3/run.c b/fs/ntfs3/run.c index 395b20492525..c0324cdc174d 100644 --- a/fs/ntfs3/run.c +++ b/fs/ntfs3/run.c @@ -454,7 +454,7 @@ requires_new_range: /* * If existing range fits then were done. - * Otherwise extend found one and fall back to range jocode. + * Otherwise extend found one and fall back to range join code. */ if (r->vcn + r->len < vcn + len) r->len += len - ((r->vcn + r->len) - vcn); @@ -482,7 +482,8 @@ requires_new_range: return true; } -/* run_collapse_range +/* + * run_collapse_range * * Helper for attr_collapse_range(), * which is helper for fallocate(collapse_range). @@ -493,8 +494,9 @@ bool run_collapse_range(struct runs_tree *run, CLST vcn, CLST len, CLST sub) struct ntfs_run *r, *e, *eat_start, *eat_end; CLST end; - if (WARN_ON(!run_lookup(run, vcn, &index))) - return true; /* Should never be here. */ + if (!run_lookup(run, vcn, &index) && index >= run->count) { + return true; + } e = run->runs + run->count; r = run->runs + index; @@ -560,13 +562,13 @@ bool run_collapse_range(struct runs_tree *run, CLST vcn, CLST len, CLST sub) * Helper for attr_insert_range(), * which is helper for fallocate(insert_range). */ -bool run_insert_range(struct runs_tree *run, CLST vcn, CLST len) +int run_insert_range(struct runs_tree *run, CLST vcn, CLST len) { size_t index; struct ntfs_run *r, *e; if (WARN_ON(!run_lookup(run, vcn, &index))) - return false; /* Should never be here. */ + return -EINVAL; /* Should never be here. */ e = run->runs + run->count; r = run->runs + index; @@ -588,13 +590,49 @@ bool run_insert_range(struct runs_tree *run, CLST vcn, CLST len) r->len = len1; if (!run_add_entry(run, vcn + len, lcn2, len2, false)) - return false; + return -ENOMEM; } if (!run_add_entry(run, vcn, SPARSE_LCN, len, false)) - return false; + return -ENOMEM; - return true; + return 0; +} + +/* run_insert_range_da + * + * Helper for attr_insert_range(), + * which is helper for fallocate(insert_range). + */ +int run_insert_range_da(struct runs_tree *run, CLST vcn, CLST len) +{ + struct ntfs_run *r, *r0 = NULL, *e = run->runs + run->count; + ; + + for (r = run->runs; r < e; r++) { + CLST end = r->vcn + r->len; + + if (vcn >= end) + continue; + + if (!r0 && r->vcn < vcn) { + r0 = r; + } else { + r->vcn += len; + } + } + + if (r0) { + /* split fragment. */ + CLST len1 = vcn - r0->vcn; + CLST len2 = r0->len - len1; + + r0->len = len1; + if (!run_add_entry(run, vcn + len, SPARSE_LCN, len2, false)) + return -ENOMEM; + } + + return 0; } /* @@ -1131,11 +1169,14 @@ int run_unpack_ex(struct runs_tree *run, struct ntfs_sb_info *sbi, CLST ino, struct rw_semaphore *lock = is_mounted(sbi) ? &sbi->mft.ni->file.run_lock : NULL; - if (lock) - down_read(lock); - ntfs_refresh_zone(sbi); - if (lock) - up_read(lock); + if (lock) { + if (down_read_trylock(lock)) { + ntfs_refresh_zone(sbi); + up_read(lock); + } + } else { + ntfs_refresh_zone(sbi); + } } up_write(&wnd->rw_lock); if (err) @@ -1206,3 +1247,97 @@ int run_clone(const struct runs_tree *run, struct runs_tree *new_run) new_run->count = run->count; return 0; } + +/* + * run_remove_range + * + */ +bool run_remove_range(struct runs_tree *run, CLST vcn, CLST len, CLST *done) +{ + size_t index, eat; + struct ntfs_run *r, *e, *eat_start, *eat_end; + CLST end, d; + + *done = 0; + + /* Fast check. */ + if (!run->count) + return true; + + if (!run_lookup(run, vcn, &index) && index >= run->count) { + /* No entries in this run. */ + return true; + } + + + e = run->runs + run->count; + r = run->runs + index; + end = vcn + len; + + if (vcn > r->vcn) { + CLST r_end = r->vcn + r->len; + d = vcn - r->vcn; + + if (r_end > end) { + /* Remove a middle part, split. */ + *done += len; + r->len = d; + return run_add_entry(run, end, r->lcn, r_end - end, + false); + } + /* Remove tail of run .*/ + *done += r->len - d; + r->len = d; + r += 1; + } + + eat_start = r; + eat_end = r; + + for (; r < e; r++) { + if (r->vcn >= end) + continue; + + if (r->vcn + r->len <= end) { + /* Eat this run. */ + *done += r->len; + eat_end = r + 1; + continue; + } + + d = end - r->vcn; + *done += d; + if (r->lcn != SPARSE_LCN) + r->lcn += d; + r->len -= d; + r->vcn = end; + } + + eat = eat_end - eat_start; + memmove(eat_start, eat_end, (e - eat_end) * sizeof(*r)); + run->count -= eat; + + return true; +} + +CLST run_len(const struct runs_tree *run) +{ + const struct ntfs_run *r, *e; + CLST len = 0; + + for (r = run->runs, e = r + run->count; r < e; r++) { + len += r->len; + } + + return len; +} + +CLST run_get_max_vcn(const struct runs_tree *run) +{ + const struct ntfs_run *r; + if (!run->count) + return 0; + + r = run->runs + run->count - 1; + return r->vcn + r->len; +} diff --git a/fs/ntfs3/super.c b/fs/ntfs3/super.c index 8b0cf0ed4f72..27411203082a 100644 --- a/fs/ntfs3/super.c +++ b/fs/ntfs3/super.c @@ -58,9 +58,9 @@ #include #include #include -#include #include #include +#include #include #include #include @@ -264,9 +264,13 @@ enum Opt { Opt_windows_names, Opt_showmeta, Opt_acl, + Opt_acl_bool, Opt_iocharset, Opt_prealloc, + Opt_prealloc_bool, Opt_nocase, + Opt_delalloc, + Opt_delalloc_bool, Opt_err, }; @@ -285,10 +289,14 @@ static const struct fs_parameter_spec ntfs_fs_parameters[] = { fsparam_flag("hide_dot_files", Opt_hide_dot_files), fsparam_flag("windows_names", Opt_windows_names), fsparam_flag("showmeta", Opt_showmeta), - fsparam_flag_no("acl", Opt_acl), + fsparam_flag("acl", Opt_acl), + fsparam_bool("acl", Opt_acl_bool), fsparam_string("iocharset", Opt_iocharset), - fsparam_flag_no("prealloc", Opt_prealloc), + fsparam_flag("prealloc", Opt_prealloc), + fsparam_bool("prealloc", Opt_prealloc_bool), fsparam_flag("nocase", Opt_nocase), + fsparam_flag("delalloc", Opt_delalloc), + fsparam_bool("delalloc", Opt_delalloc_bool), {} }; // clang-format on @@ -379,15 +387,17 @@ static int ntfs_fs_parse_param(struct fs_context *fc, case Opt_showmeta: opts->showmeta = 1; break; - case Opt_acl: - if (!result.negated) + case Opt_acl_bool: + if (result.boolean) { + fallthrough; + case Opt_acl: #ifdef CONFIG_NTFS3_FS_POSIX_ACL fc->sb_flags |= SB_POSIXACL; #else return invalf( fc, "ntfs3: Support for ACL not compiled in!"); #endif - else + } else fc->sb_flags &= ~SB_POSIXACL; break; case Opt_iocharset: @@ -396,11 +406,20 @@ static int ntfs_fs_parse_param(struct fs_context *fc, param->string = NULL; break; case Opt_prealloc: - opts->prealloc = !result.negated; + opts->prealloc = 1; + break; + case Opt_prealloc_bool: + opts->prealloc = result.boolean; break; case Opt_nocase: opts->nocase = 1; break; + case Opt_delalloc: + opts->delalloc = 1; + break; + case Opt_delalloc_bool: + opts->delalloc = result.boolean; + break; default: /* Should not be here unless we forget add case. */ return -EINVAL; @@ -674,7 +693,7 @@ static noinline void ntfs3_put_sbi(struct ntfs_sb_info *sbi) sbi->volume.ni = NULL; } - ntfs_update_mftmirr(sbi, 0); + ntfs_update_mftmirr(sbi); indx_clear(&sbi->security.index_sii); indx_clear(&sbi->security.index_sdh); @@ -705,9 +724,7 @@ static void ntfs_put_super(struct super_block *sb) ntfs_set_state(sbi, NTFS_DIRTY_CLEAR); if (sbi->options) { - unload_nls(sbi->options->nls); - kfree(sbi->options->nls_name); - kfree(sbi->options); + put_mount_options(sbi->options); sbi->options = NULL; } @@ -719,14 +736,22 @@ static int ntfs_statfs(struct dentry *dentry, struct kstatfs *buf) struct super_block *sb = dentry->d_sb; struct ntfs_sb_info *sbi = sb->s_fs_info; struct wnd_bitmap *wnd = &sbi->used.bitmap; + CLST da_clusters = ntfs_get_da(sbi); buf->f_type = sb->s_magic; - buf->f_bsize = sbi->cluster_size; + buf->f_bsize = buf->f_frsize = sbi->cluster_size; buf->f_blocks = wnd->nbits; - buf->f_bfree = buf->f_bavail = wnd_zeroes(wnd); + buf->f_bfree = wnd_zeroes(wnd); + if (buf->f_bfree > da_clusters) { + buf->f_bfree -= da_clusters; + } else { + buf->f_bfree = 0; + } + buf->f_bavail = buf->f_bfree; + buf->f_fsid.val[0] = sbi->volume.ser_num; - buf->f_fsid.val[1] = (sbi->volume.ser_num >> 32); + buf->f_fsid.val[1] = sbi->volume.ser_num >> 32; buf->f_namelen = NTFS_NAME_LEN; return 0; @@ -771,6 +796,8 @@ static int ntfs_show_options(struct seq_file *m, struct dentry *root) seq_puts(m, ",prealloc"); if (opts->nocase) seq_puts(m, ",nocase"); + if (opts->delalloc) + seq_puts(m, ",delalloc"); return 0; } @@ -823,7 +850,12 @@ static int ntfs_sync_fs(struct super_block *sb, int wait) if (!err) ntfs_set_state(sbi, NTFS_DIRTY_CLEAR); - ntfs_update_mftmirr(sbi, wait); + ntfs_update_mftmirr(sbi); + + if (wait) { + sync_blockdev(sb->s_bdev); + blkdev_issue_flush(sb->s_bdev); + } return err; } @@ -1076,7 +1108,7 @@ read_boot: dev_size += sector_size - 1; } - sbi->bdev_blocksize_mask = max(boot_sector_size, sector_size) - 1; + sbi->bdev_blocksize = max(boot_sector_size, sector_size); sbi->mft.lbo = mlcn << cluster_bits; sbi->mft.lbo2 = mlcn2 << cluster_bits; @@ -1253,7 +1285,6 @@ static int ntfs_fill_super(struct super_block *sb, struct fs_context *fc) } } sbi->options = options; - fc->fs_private = NULL; sb->s_flags |= SB_NODIRATIME; sb->s_magic = 0x7366746e; // "ntfs" sb->s_op = &ntfs_sops; @@ -1652,9 +1683,7 @@ load_root: */ struct buffer_head *bh0 = sb_getblk(sb, 0); if (bh0) { - if (buffer_locked(bh0)) - __wait_on_buffer(bh0); - + wait_on_buffer(bh0); lock_buffer(bh0); memcpy(bh0->b_data, boot2, sizeof(*boot2)); set_buffer_uptodate(bh0); @@ -1679,9 +1708,7 @@ put_inode_out: out: /* sbi->options == options */ if (options) { - unload_nls(options->nls); - kfree(options->nls_name); - kfree(options); + put_mount_options(sbi->options); sbi->options = NULL; } diff --git a/fs/ntfs3/xattr.c b/fs/ntfs3/xattr.c index b96e6ac90e9f..3fffda784892 100644 --- a/fs/ntfs3/xattr.c +++ b/fs/ntfs3/xattr.c @@ -460,7 +460,7 @@ update_ea: new_sz = size; err = attr_set_size(ni, ATTR_EA, NULL, 0, &ea_run, new_sz, &new_sz, - false, NULL); + false); if (err) goto out; diff --git a/fs/overlayfs/namei.c b/fs/overlayfs/namei.c index cda26bdef3b9..afb7019fd43a 100644 --- a/fs/overlayfs/namei.c +++ b/fs/overlayfs/namei.c @@ -159,6 +159,18 @@ invalid: goto out; } +bool ovl_uuid_match(struct ovl_fs *ofs, const struct super_block *sb, + const uuid_t *uuid) +{ + /* + * Make sure that the stored uuid matches the uuid of the lower + * layer where file handle will be decoded. + * In case of uuid=off option just make sure that stored uuid is null. + */ + return ovl_origin_uuid(ofs) ? uuid_equal(uuid, &sb->s_uuid) : + uuid_is_null(uuid); +} + struct dentry *ovl_decode_real_fh(struct ovl_fs *ofs, struct ovl_fh *fh, struct vfsmount *mnt, bool connected) { @@ -168,14 +180,7 @@ struct dentry *ovl_decode_real_fh(struct ovl_fs *ofs, struct ovl_fh *fh, if (!capable(CAP_DAC_READ_SEARCH)) return NULL; - /* - * Make sure that the stored uuid matches the uuid of the lower - * layer where file handle will be decoded. - * In case of uuid=off option just make sure that stored uuid is null. - */ - if (ovl_origin_uuid(ofs) ? - !uuid_equal(&fh->fb.uuid, &mnt->mnt_sb->s_uuid) : - !uuid_is_null(&fh->fb.uuid)) + if (!ovl_uuid_match(ofs, mnt->mnt_sb, &fh->fb.uuid)) return NULL; bytes = (fh->fb.len - offsetof(struct ovl_fb, fid)); diff --git a/fs/overlayfs/overlayfs.h b/fs/overlayfs/overlayfs.h index 315882a360ce..cad2055ebf18 100644 --- a/fs/overlayfs/overlayfs.h +++ b/fs/overlayfs/overlayfs.h @@ -710,6 +710,8 @@ static inline int ovl_check_fh_len(struct ovl_fh *fh, int fh_len) return ovl_check_fb_len(&fh->fb, fh_len - OVL_FH_WIRE_OFFSET); } +bool ovl_uuid_match(struct ovl_fs *ofs, const struct super_block *sb, + const uuid_t *uuid); struct dentry *ovl_decode_real_fh(struct ovl_fs *ofs, struct ovl_fh *fh, struct vfsmount *mnt, bool connected); int ovl_check_origin_fh(struct ovl_fs *ofs, struct ovl_fh *fh, bool connected, diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index ba9146f22a2c..c9f166a1390a 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -940,7 +940,7 @@ static bool ovl_lower_uuid_ok(struct ovl_fs *ofs, const uuid_t *uuid) * disable lower file handle decoding on all of them. */ if (ofs->fs[i].is_lower && - uuid_equal(&ofs->fs[i].sb->s_uuid, uuid)) { + ovl_uuid_match(ofs, ofs->fs[i].sb, uuid)) { ofs->fs[i].bad_uuid = true; return false; } @@ -952,6 +952,7 @@ static bool ovl_lower_uuid_ok(struct ovl_fs *ofs, const uuid_t *uuid) static int ovl_get_fsid(struct ovl_fs *ofs, const struct path *path) { struct super_block *sb = path->mnt->mnt_sb; + const uuid_t *uuid = ovl_origin_uuid(ofs) ? &sb->s_uuid : &uuid_null; unsigned int i; dev_t dev; int err; @@ -963,7 +964,7 @@ static int ovl_get_fsid(struct ovl_fs *ofs, const struct path *path) return i; } - if (!ovl_lower_uuid_ok(ofs, &sb->s_uuid)) { + if (!ovl_lower_uuid_ok(ofs, uuid)) { bad_uuid = true; if (ofs->config.xino == OVL_XINO_AUTO) { ofs->config.xino = OVL_XINO_OFF; @@ -975,9 +976,8 @@ static int ovl_get_fsid(struct ovl_fs *ofs, const struct path *path) warn = true; } if (warn) { - pr_warn("%s uuid detected in lower fs '%pd2', falling back to xino=%s,index=off,nfs_export=off.\n", - uuid_is_null(&sb->s_uuid) ? "null" : - "conflicting", + pr_warn("%s uuid in non-single lower fs '%pd2', falling back to xino=%s,index=off,nfs_export=off.\n", + uuid_is_null(uuid) ? "null" : "conflicting", path->dentry, ovl_xino_mode(&ofs->config)); } } @@ -1469,10 +1469,7 @@ static int ovl_fill_super_creds(struct fs_context *fc, struct super_block *sb) if (!ovl_upper_mnt(ofs)) sb->s_flags |= SB_RDONLY; - if (!ovl_origin_uuid(ofs) && ofs->numfs > 1) { - pr_warn("The uuid=off requires a single fs for lower and upper, falling back to uuid=null.\n"); - ofs->config.uuid = OVL_UUID_NULL; - } else if (ovl_has_fsid(ofs) && ovl_upper_mnt(ofs)) { + if (ovl_has_fsid(ofs) && ovl_upper_mnt(ofs)) { /* Use per instance persistent uuid/fsid */ ovl_init_uuid_xattr(sb, ofs, &ctx->upper); } diff --git a/fs/pipe.c b/fs/pipe.c index 9e6a01475815..22647f50b286 100644 --- a/fs/pipe.c +++ b/fs/pipe.c @@ -1481,10 +1481,24 @@ static struct file_system_type pipe_fs_type = { }; #ifdef CONFIG_SYSCTL -static SYSCTL_USER_TO_KERN_UINT_CONV(_pipe_maxsz, round_pipe_size) -static SYSCTL_UINT_CONV_CUSTOM(_pipe_maxsz, - sysctl_user_to_kern_uint_conv_pipe_maxsz, - sysctl_kern_to_user_uint_conv, true) + +static ulong round_pipe_size_ul(ulong size) +{ + return round_pipe_size(size); +} + +static int u2k_pipe_maxsz(const ulong *u_ptr, uint *k_ptr) +{ + return proc_uint_u2k_conv_uop(u_ptr, k_ptr, round_pipe_size_ul); +} + +static int do_proc_uint_conv_pipe_maxsz(ulong *u_ptr, uint *k_ptr, + int dir, const struct ctl_table *table) +{ + return proc_uint_conv(u_ptr, k_ptr, dir, table, true, + u2k_pipe_maxsz, + proc_uint_k2u_conv); +} static int proc_dopipe_max_size(const struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) diff --git a/include/linux/ceph/ceph_fs.h b/include/linux/ceph/ceph_fs.h index 08e5dbe15ca4..69ac3e55a3fe 100644 --- a/include/linux/ceph/ceph_fs.h +++ b/include/linux/ceph/ceph_fs.h @@ -89,8 +89,9 @@ struct ceph_dir_layout { } __attribute__ ((packed)); /* crypto algorithms */ -#define CEPH_CRYPTO_NONE 0x0 -#define CEPH_CRYPTO_AES 0x1 +#define CEPH_CRYPTO_NONE 0x0 +#define CEPH_CRYPTO_AES 0x1 +#define CEPH_CRYPTO_AES256KRB5 0x2 /* AES256-CTS-HMAC384-192 */ #define CEPH_AES_IV "cephsageyudagreg" diff --git a/include/linux/fs.h b/include/linux/fs.h index a2af5ddd5323..8b3dd145b25e 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -3525,7 +3525,6 @@ ssize_t simple_attr_write(struct file *file, const char __user *buf, ssize_t simple_attr_write_signed(struct file *file, const char __user *buf, size_t len, loff_t *ppos); -struct ctl_table; int __init list_bdev_fs_names(char *buf, size_t size); #define __FMODE_EXEC ((__force int) FMODE_EXEC) diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 94a03591990c..d2e48fa5f72e 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -16,8 +16,6 @@ #include #include -struct ctl_table; -struct user_struct; struct mmu_gather; struct node; diff --git a/include/linux/intel_rapl.h b/include/linux/intel_rapl.h index f479ef5b3341..fa1f328d6712 100644 --- a/include/linux/intel_rapl.h +++ b/include/linux/intel_rapl.h @@ -152,7 +152,7 @@ struct rapl_if_priv { union rapl_reg reg_unit; union rapl_reg regs[RAPL_DOMAIN_MAX][RAPL_DOMAIN_REG_MAX]; int limits[RAPL_DOMAIN_MAX]; - int (*read_raw)(int id, struct reg_action *ra, bool atomic); + int (*read_raw)(int id, struct reg_action *ra, bool pmu_ctx); int (*write_raw)(int id, struct reg_action *ra); void *defaults; void *rpi; diff --git a/include/linux/printk.h b/include/linux/printk.h index 45c663124c9b..63d516c873b4 100644 --- a/include/linux/printk.h +++ b/include/linux/printk.h @@ -78,7 +78,6 @@ extern void console_verbose(void); /* strlen("ratelimit") + 1 */ #define DEVKMSG_STR_MAX_SIZE 10 extern char devkmsg_log_str[DEVKMSG_STR_MAX_SIZE]; -struct ctl_table; extern int suppress_printk; diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h index 288fe0055cd5..2886fbceb5d6 100644 --- a/include/linux/sysctl.h +++ b/include/linux/sysctl.h @@ -59,7 +59,6 @@ extern const int sysctl_vals[]; #define SYSCTL_LONG_ONE ((void *)&sysctl_long_vals[1]) #define SYSCTL_LONG_MAX ((void *)&sysctl_long_vals[2]) -#define SYSCTL_CONV_IDENTITY(val) (val) /** * * "dir" originates from read_iter (dir = 0) or write_iter (dir = 1) @@ -73,107 +72,6 @@ extern const int sysctl_vals[]; #define SYSCTL_USER_TO_KERN(dir) (!!(dir)) #define SYSCTL_KERN_TO_USER(dir) (!dir) -#define SYSCTL_USER_TO_KERN_INT_CONV(name, u_ptr_op) \ -int sysctl_user_to_kern_int_conv##name(const bool *negp, \ - const unsigned long *u_ptr,\ - int *k_ptr) \ -{ \ - unsigned long u = u_ptr_op(*u_ptr); \ - if (*negp) { \ - if (u > (unsigned long) INT_MAX + 1) \ - return -EINVAL; \ - WRITE_ONCE(*k_ptr, -u); \ - } else { \ - if (u > (unsigned long) INT_MAX) \ - return -EINVAL; \ - WRITE_ONCE(*k_ptr, u); \ - } \ - return 0; \ -} - -#define SYSCTL_KERN_TO_USER_INT_CONV(name, k_ptr_op) \ -int sysctl_kern_to_user_int_conv##name(bool *negp, \ - unsigned long *u_ptr, \ - const int *k_ptr) \ -{ \ - int val = READ_ONCE(*k_ptr); \ - if (val < 0) { \ - *negp = true; \ - *u_ptr = -k_ptr_op((unsigned long)val); \ - } else { \ - *negp = false; \ - *u_ptr = k_ptr_op((unsigned long)val); \ - } \ - return 0; \ -} - -/** - * To range check on a converted value, use a temp k_ptr - * When checking range, value should be within (tbl->extra1, tbl->extra2) - */ -#define SYSCTL_INT_CONV_CUSTOM(name, user_to_kern, kern_to_user, \ - k_ptr_range_check) \ -int do_proc_int_conv##name(bool *negp, unsigned long *u_ptr, int *k_ptr,\ - int dir, const struct ctl_table *tbl) \ -{ \ - if (SYSCTL_KERN_TO_USER(dir)) \ - return kern_to_user(negp, u_ptr, k_ptr); \ - \ - if (k_ptr_range_check) { \ - int tmp_k, ret; \ - if (!tbl) \ - return -EINVAL; \ - ret = user_to_kern(negp, u_ptr, &tmp_k); \ - if (ret) \ - return ret; \ - if ((tbl->extra1 && *(int *)tbl->extra1 > tmp_k) || \ - (tbl->extra2 && *(int *)tbl->extra2 < tmp_k)) \ - return -EINVAL; \ - WRITE_ONCE(*k_ptr, tmp_k); \ - } else \ - return user_to_kern(negp, u_ptr, k_ptr); \ - return 0; \ -} - -#define SYSCTL_USER_TO_KERN_UINT_CONV(name, u_ptr_op) \ -int sysctl_user_to_kern_uint_conv##name(const unsigned long *u_ptr,\ - unsigned int *k_ptr) \ -{ \ - unsigned long u = u_ptr_op(*u_ptr); \ - if (u > UINT_MAX) \ - return -EINVAL; \ - WRITE_ONCE(*k_ptr, u); \ - return 0; \ -} - -#define SYSCTL_UINT_CONV_CUSTOM(name, user_to_kern, kern_to_user, \ - k_ptr_range_check) \ -int do_proc_uint_conv##name(unsigned long *u_ptr, unsigned int *k_ptr, \ - int dir, const struct ctl_table *tbl) \ -{ \ - if (SYSCTL_KERN_TO_USER(dir)) \ - return kern_to_user(u_ptr, k_ptr); \ - \ - if (k_ptr_range_check) { \ - unsigned int tmp_k; \ - int ret; \ - if (!tbl) \ - return -EINVAL; \ - ret = user_to_kern(u_ptr, &tmp_k); \ - if (ret) \ - return ret; \ - if ((tbl->extra1 && \ - *(unsigned int *)tbl->extra1 > tmp_k) || \ - (tbl->extra2 && \ - *(unsigned int *)tbl->extra2 < tmp_k)) \ - return -ERANGE; \ - WRITE_ONCE(*k_ptr, tmp_k); \ - } else \ - return user_to_kern(u_ptr, k_ptr); \ - return 0; \ -} - - extern const unsigned long sysctl_long_vals[]; typedef int proc_handler(const struct ctl_table *ctl, int write, void *buffer, @@ -182,6 +80,7 @@ typedef int proc_handler(const struct ctl_table *ctl, int write, void *buffer, int proc_dostring(const struct ctl_table *, int, void *, size_t *, loff_t *); int proc_dobool(const struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); + int proc_dointvec(const struct ctl_table *, int, void *, size_t *, loff_t *); int proc_dointvec_minmax(const struct ctl_table *table, int dir, void *buffer, size_t *lenp, loff_t *ppos); @@ -189,6 +88,15 @@ int proc_dointvec_conv(const struct ctl_table *table, int dir, void *buffer, size_t *lenp, loff_t *ppos, int (*conv)(bool *negp, unsigned long *u_ptr, int *k_ptr, int dir, const struct ctl_table *table)); +int proc_int_k2u_conv_kop(ulong *u_ptr, const int *k_ptr, bool *negp, + ulong (*k_ptr_op)(const ulong)); +int proc_int_u2k_conv_uop(const ulong *u_ptr, int *k_ptr, const bool *negp, + ulong (*u_ptr_op)(const ulong)); +int proc_int_conv(bool *negp, ulong *u_ptr, int *k_ptr, int dir, + const struct ctl_table *tbl, bool k_ptr_range_check, + int (*user_to_kern)(const bool *negp, const ulong *u_ptr, int *k_ptr), + int (*kern_to_user)(bool *negp, ulong *u_ptr, const int *k_ptr)); + int proc_douintvec(const struct ctl_table *, int, void *, size_t *, loff_t *); int proc_douintvec_minmax(const struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); @@ -196,6 +104,13 @@ int proc_douintvec_conv(const struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos, int (*conv)(unsigned long *lvalp, unsigned int *valp, int write, const struct ctl_table *table)); +int proc_uint_k2u_conv(ulong *u_ptr, const uint *k_ptr); +int proc_uint_u2k_conv_uop(const ulong *u_ptr, uint *k_ptr, + ulong (*u_ptr_op)(const ulong)); +int proc_uint_conv(ulong *u_ptr, uint *k_ptr, int dir, + const struct ctl_table *tbl, bool k_ptr_range_check, + int (*user_to_kern)(const ulong *u_ptr, uint *k_ptr), + int (*kern_to_user)(ulong *u_ptr, const uint *k_ptr)); int proc_dou8vec_minmax(const struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); @@ -206,7 +121,6 @@ int proc_doulongvec_minmax_conv(const struct ctl_table *table, int dir, int proc_do_large_bitmap(const struct ctl_table *, int, void *, size_t *, loff_t *); int proc_do_static_key(const struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); -int sysctl_kern_to_user_uint_conv(unsigned long *u_ptr, const unsigned int *k_ptr); /* * Register a set of sysctl names by calling register_sysctl diff --git a/include/net/ax25.h b/include/net/ax25.h index ad3d7626130e..9fc6a6657266 100644 --- a/include/net/ax25.h +++ b/include/net/ax25.h @@ -211,8 +211,6 @@ typedef struct { unsigned short slave_timeout; /* when? */ } ax25_dama_info; -struct ctl_table; - typedef struct ax25_dev { struct list_head list; diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index 5fdea5682756..85fbf1801cbe 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -4,9 +4,9 @@ */ #include #include +#include #if defined(CONFIG_PRINTK) && defined(CONFIG_SYSCTL) -struct ctl_table; void __init printk_sysctl_init(void); int devkmsg_sysctl_set_loglvl(const struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); diff --git a/kernel/printk/sysctl.c b/kernel/printk/sysctl.c index da77f3f5c1fe..f15732e93c2e 100644 --- a/kernel/printk/sysctl.c +++ b/kernel/printk/sysctl.c @@ -3,7 +3,6 @@ * sysctl.c: General linux system control interface */ -#include #include #include #include diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 2cd767b9680e..9d3a666ffde1 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -354,29 +354,221 @@ static void proc_put_char(void **buf, size_t *size, char c) } } -static SYSCTL_USER_TO_KERN_INT_CONV(, SYSCTL_CONV_IDENTITY) -static SYSCTL_KERN_TO_USER_INT_CONV(, SYSCTL_CONV_IDENTITY) - -static SYSCTL_INT_CONV_CUSTOM(, sysctl_user_to_kern_int_conv, - sysctl_kern_to_user_int_conv, false) -static SYSCTL_INT_CONV_CUSTOM(_minmax, sysctl_user_to_kern_int_conv, - sysctl_kern_to_user_int_conv, true) - - -static SYSCTL_USER_TO_KERN_UINT_CONV(, SYSCTL_CONV_IDENTITY) - -int sysctl_kern_to_user_uint_conv(unsigned long *u_ptr, - const unsigned int *k_ptr) +/** + * proc_uint_u2k_conv_uop - Assign user value to a kernel pointer + * + * @u_ptr: pointer to user space variable + * @k_ptr: pointer to kernel variable + * @u_ptr_op: execute this function before assigning to k_ptr + * + * Uses WRITE_ONCE to assign value to k_ptr. Executes u_ptr_op if + * not NULL. Check that the values are less than UINT_MAX to avoid + * having to support wrap around from userspace. + * + * returns 0 on success. + */ +int proc_uint_u2k_conv_uop(const ulong *u_ptr, uint *k_ptr, + ulong (*u_ptr_op)(const ulong)) { - unsigned int val = READ_ONCE(*k_ptr); - *u_ptr = (unsigned long)val; + ulong u = u_ptr_op ? u_ptr_op(*u_ptr) : *u_ptr; + + if (u > UINT_MAX) + return -EINVAL; + WRITE_ONCE(*k_ptr, u); return 0; } -static SYSCTL_UINT_CONV_CUSTOM(, sysctl_user_to_kern_uint_conv, - sysctl_kern_to_user_uint_conv, false) -static SYSCTL_UINT_CONV_CUSTOM(_minmax, sysctl_user_to_kern_uint_conv, - sysctl_kern_to_user_uint_conv, true) +/** + * proc_uint_k2u_conv - Assign kernel value to a user space pointer + * + * @u_ptr: pointer to user space variable + * @k_ptr: pointer to kernel variable + * + * Uses READ_ONCE to assign value to u_ptr. + * + * returns 0 on success. + */ +int proc_uint_k2u_conv(ulong *u_ptr, const uint *k_ptr) +{ + uint val = READ_ONCE(*k_ptr); + *u_ptr = (ulong)val; + return 0; +} + +/** + * proc_uint_conv - Change user or kernel pointer based on direction + * + * @u_ptr: pointer to user variable + * @k_ptr: pointer to kernel variable + * @dir: %TRUE if this is a write to the sysctl file + * @tbl: the sysctl table + * @k_ptr_range_check: Check range for k_ptr when %TRUE + * @user_to_kern: Callback used to assign value from user to kernel var + * @kern_to_user: Callback used to assign value from kernel to user var + * + * When direction is kernel to user, then the u_ptr is modified. + * When direction is user to kernel, then the k_ptr is modified. + * + * Returns 0 on success + */ +int proc_uint_conv(ulong *u_ptr, uint *k_ptr, int dir, + const struct ctl_table *tbl, bool k_ptr_range_check, + int (*user_to_kern)(const ulong *u_ptr, uint *k_ptr), + int (*kern_to_user)(ulong *u_ptr, const uint *k_ptr)) +{ + if (SYSCTL_KERN_TO_USER(dir)) + return kern_to_user(u_ptr, k_ptr); + + if (k_ptr_range_check) { + uint tmp_k; + int ret; + + if (!tbl) + return -EINVAL; + ret = user_to_kern(u_ptr, &tmp_k); + if (ret) + return ret; + if ((tbl->extra1 && + *(uint *)tbl->extra1 > tmp_k) || + (tbl->extra2 && + *(uint *)tbl->extra2 < tmp_k)) + return -ERANGE; + WRITE_ONCE(*k_ptr, tmp_k); + } else + return user_to_kern(u_ptr, k_ptr); + return 0; +} + +static int proc_uint_u2k_conv(const ulong *u_ptr, uint *k_ptr) +{ + return proc_uint_u2k_conv_uop(u_ptr, k_ptr, NULL); +} + +static int do_proc_uint_conv(ulong *u_ptr, uint *k_ptr, int dir, + const struct ctl_table *tbl) +{ + return proc_uint_conv(u_ptr, k_ptr, dir, tbl, false, + proc_uint_u2k_conv, proc_uint_k2u_conv); +} + +static int do_proc_uint_conv_minmax(ulong *u_ptr, uint *k_ptr, int dir, + const struct ctl_table *tbl) +{ + return proc_uint_conv(u_ptr, k_ptr, dir, tbl, true, + proc_uint_u2k_conv, proc_uint_k2u_conv); +} + +/** + * proc_int_k2u_conv_kop - Assign kernel value to a user space pointer + * @u_ptr: pointer to user space variable + * @k_ptr: pointer to kernel variable + * @negp: assigned %TRUE if the converted kernel value is negative; + * %FALSE otherweise + * @k_ptr_op: execute this function before assigning to u_ptr + * + * Uses READ_ONCE to get value from k_ptr. Executes k_ptr_op before assigning + * to u_ptr if not NULL. Does **not** check for overflow. + * + * Returns: 0 on success. + */ +int proc_int_k2u_conv_kop(ulong *u_ptr, const int *k_ptr, bool *negp, + ulong (*k_ptr_op)(const ulong)) +{ + int val = READ_ONCE(*k_ptr); + + if (val < 0) { + *negp = true; + *u_ptr = k_ptr_op ? -k_ptr_op((ulong)val) : -(ulong)val; + } else { + *negp = false; + *u_ptr = k_ptr_op ? k_ptr_op((ulong)val) : (ulong) val; + } + return 0; +} + +/** + * proc_int_u2k_conv_uop - Assign user value to a kernel pointer + * @u_ptr: pointer to user space variable + * @k_ptr: pointer to kernel variable + * @negp: If %TRUE, the converted user value is made negative. + * @u_ptr_op: execute this function before assigning to k_ptr + * + * Uses WRITE_ONCE to assign value to k_ptr. Executes u_ptr_op if + * not NULL. Check for overflow with UINT_MAX. + * + * Returns: 0 on success. + */ +int proc_int_u2k_conv_uop(const ulong *u_ptr, int *k_ptr, const bool *negp, + ulong (*u_ptr_op)(const ulong)) +{ + ulong u = u_ptr_op ? u_ptr_op(*u_ptr) : *u_ptr; + + if (*negp) { + if (u > (ulong) INT_MAX + 1) + return -EINVAL; + WRITE_ONCE(*k_ptr, -u); + } else { + if (u > (ulong) INT_MAX) + return -EINVAL; + WRITE_ONCE(*k_ptr, u); + } + return 0; +} + +int proc_int_conv(bool *negp, ulong *u_ptr, int *k_ptr, int dir, + const struct ctl_table *tbl, bool k_ptr_range_check, + int (*user_to_kern)(const bool *negp, const ulong *u_ptr, int *k_ptr), + int (*kern_to_user)(bool *negp, ulong *u_ptr, const int *k_ptr)) +{ + if (SYSCTL_KERN_TO_USER(dir)) + return kern_to_user(negp, u_ptr, k_ptr); + + if (k_ptr_range_check) { + int tmp_k, ret; + + if (!tbl) + return -EINVAL; + ret = user_to_kern(negp, u_ptr, &tmp_k); + if (ret) + return ret; + if ((tbl->extra1 && *(int *)tbl->extra1 > tmp_k) || + (tbl->extra2 && *(int *)tbl->extra2 < tmp_k)) + return -EINVAL; + WRITE_ONCE(*k_ptr, tmp_k); + } else + return user_to_kern(negp, u_ptr, k_ptr); + return 0; +} + + + +static int sysctl_user_to_kern_int_conv(const bool *negp, const ulong *u_ptr, + int *k_ptr) +{ + return proc_int_u2k_conv_uop(u_ptr, k_ptr, negp, NULL); +} + +static int sysctl_kern_to_user_int_conv(bool *negp, ulong *u_ptr, const int *k_ptr) +{ + return proc_int_k2u_conv_kop(u_ptr, k_ptr, negp, NULL); +} + +static int do_proc_int_conv(bool *negp, unsigned long *u_ptr, int *k_ptr, + int dir, const struct ctl_table *tbl) +{ + return proc_int_conv(negp, u_ptr, k_ptr, dir, tbl, false, + sysctl_user_to_kern_int_conv, + sysctl_kern_to_user_int_conv); + +} + +static int do_proc_int_conv_minmax(bool *negp, unsigned long *u_ptr, int *k_ptr, + int dir, const struct ctl_table *tbl) +{ + return proc_int_conv(negp, u_ptr, k_ptr, dir, tbl, true, + sysctl_user_to_kern_int_conv, + sysctl_kern_to_user_int_conv); +} static const char proc_wspace_sep[] = { ' ', '\t', '\n' }; @@ -568,6 +760,22 @@ static int do_proc_douintvec(const struct ctl_table *table, int dir, return do_proc_douintvec_r(table, buffer, lenp, ppos, conv); } +/** + * proc_douintvec_conv - read a vector of unsigned ints with a custom converter + * + * @table: the sysctl table + * @dir: %TRUE if this is a write to the sysctl file + * @buffer: the user buffer + * @lenp: the size of the user buffer + * @ppos: file position + * @conv: Custom converter call back + * + * Reads/writes up to table->maxlen/sizeof(unsigned int) unsigned integer + * values from/to the user buffer, treated as an ASCII string. Negative + * strings are not allowed. + * + * Returns 0 on success + */ int proc_douintvec_conv(const struct ctl_table *table, int dir, void *buffer, size_t *lenp, loff_t *ppos, int (*conv)(unsigned long *u_ptr, unsigned int *k_ptr, @@ -576,7 +784,6 @@ int proc_douintvec_conv(const struct ctl_table *table, int dir, void *buffer, return do_proc_douintvec(table, dir, buffer, lenp, ppos, conv); } - /** * proc_dobool - read/write a bool * @table: the sysctl table @@ -692,10 +899,10 @@ int proc_dointvec_minmax(const struct ctl_table *table, int dir, * values from/to the user buffer, treated as an ASCII string. Negative * strings are not allowed. * - * This routine will ensure the values are within the range specified by - * table->extra1 (min) and table->extra2 (max). There is a final sanity - * check for UINT_MAX to avoid having to support wrap around uses from - * userspace. + * When changing the kernel variable, this routine will ensure the values + * are within the range specified by table->extra1 (min) and table->extra2 + * (max). And Check that the values are less than UINT_MAX to avoid having to + * support wrap around uses from userspace. * * Returns 0 on success or -ERANGE when range check failes and * SYSCTL_USER_TO_KERN(dir) == true @@ -862,6 +1069,22 @@ int proc_doulongvec_minmax(const struct ctl_table *table, int dir, return proc_doulongvec_minmax_conv(table, dir, buffer, lenp, ppos, 1l, 1l); } +/** + * proc_dointvec_conv - read a vector of ints with a custom converter + * @table: the sysctl table + * @dir: %TRUE if this is a write to the sysctl file + * @buffer: the user buffer + * @lenp: the size of the user buffer + * @ppos: file position + * @conv: Custom converter call back + * + * Reads/writes up to table->maxlen/sizeof(unsigned int) unsigned integer + * values from/to the user buffer, treated as an ASCII string. Negative + * strings are not allowed. + * + * Returns: 0 on success + */ + int proc_dointvec_conv(const struct ctl_table *table, int dir, void *buffer, size_t *lenp, loff_t *ppos, int (*conv)(bool *negp, unsigned long *u_ptr, int *k_ptr, @@ -1055,6 +1278,33 @@ int proc_douintvec_minmax(const struct ctl_table *table, int dir, return -ENOSYS; } +int proc_douintvec_conv(const struct ctl_table *table, int write, void *buffer, + size_t *lenp, loff_t *ppos, + int (*conv)(unsigned long *lvalp, unsigned int *valp, + int write, const struct ctl_table *table)) +{ + return -ENOSYS; +} + +int proc_uint_k2u_conv(ulong *u_ptr, const uint *k_ptr) +{ + return -ENOSYS; +} + +int proc_uint_u2k_conv_uop(const ulong *u_ptr, uint *k_ptr, + ulong (*u_ptr_op)(const ulong)) +{ + return -ENOSYS; +} + +int proc_uint_conv(ulong *u_ptr, uint *k_ptr, int dir, + const struct ctl_table *tbl, bool k_ptr_range_check, + int (*user_to_kern)(const ulong *u_ptr, uint *k_ptr), + int (*kern_to_user)(ulong *u_ptr, const uint *k_ptr)) +{ + return -ENOSYS; +} + int proc_dou8vec_minmax(const struct ctl_table *table, int dir, void *buffer, size_t *lenp, loff_t *ppos) { diff --git a/kernel/time/jiffies.c b/kernel/time/jiffies.c index d31a6d40d38d..a5c7d15fce72 100644 --- a/kernel/time/jiffies.c +++ b/kernel/time/jiffies.c @@ -100,26 +100,120 @@ void __init register_refined_jiffies(long cycles_per_second) __clocksource_register(&refined_jiffies); } -#define SYSCTL_CONV_MULT_HZ(val) ((val) * HZ) -#define SYSCTL_CONV_DIV_HZ(val) ((val) / HZ) +#ifdef CONFIG_PROC_SYSCTL +static ulong mult_hz(const ulong val) +{ + return val * HZ; +} -static SYSCTL_USER_TO_KERN_INT_CONV(_hz, SYSCTL_CONV_MULT_HZ) -static SYSCTL_KERN_TO_USER_INT_CONV(_hz, SYSCTL_CONV_DIV_HZ) -static SYSCTL_USER_TO_KERN_INT_CONV(_userhz, clock_t_to_jiffies) -static SYSCTL_KERN_TO_USER_INT_CONV(_userhz, jiffies_to_clock_t) -static SYSCTL_USER_TO_KERN_INT_CONV(_ms, msecs_to_jiffies) -static SYSCTL_KERN_TO_USER_INT_CONV(_ms, jiffies_to_msecs) +static ulong div_hz(const ulong val) +{ + return val / HZ; +} -static SYSCTL_INT_CONV_CUSTOM(_jiffies, sysctl_user_to_kern_int_conv_hz, - sysctl_kern_to_user_int_conv_hz, false) -static SYSCTL_INT_CONV_CUSTOM(_userhz_jiffies, - sysctl_user_to_kern_int_conv_userhz, - sysctl_kern_to_user_int_conv_userhz, false) -static SYSCTL_INT_CONV_CUSTOM(_ms_jiffies, sysctl_user_to_kern_int_conv_ms, - sysctl_kern_to_user_int_conv_ms, false) -static SYSCTL_INT_CONV_CUSTOM(_ms_jiffies_minmax, - sysctl_user_to_kern_int_conv_ms, - sysctl_kern_to_user_int_conv_ms, true) +static int sysctl_u2k_int_conv_hz(const bool *negp, const ulong *u_ptr, int *k_ptr) +{ + return proc_int_u2k_conv_uop(u_ptr, k_ptr, negp, mult_hz); +} + +static int sysctl_k2u_int_conv_hz(bool *negp, ulong *u_ptr, const int *k_ptr) +{ + return proc_int_k2u_conv_kop(u_ptr, k_ptr, negp, div_hz); +} + +static int sysctl_u2k_int_conv_userhz(const bool *negp, const ulong *u_ptr, int *k_ptr) +{ + return proc_int_u2k_conv_uop(u_ptr, k_ptr, negp, clock_t_to_jiffies); +} + +static ulong sysctl_jiffies_to_clock_t(const ulong val) +{ + return jiffies_to_clock_t(val); +} + +static int sysctl_k2u_int_conv_userhz(bool *negp, ulong *u_ptr, const int *k_ptr) +{ + return proc_int_k2u_conv_kop(u_ptr, k_ptr, negp, sysctl_jiffies_to_clock_t); +} + +static ulong sysctl_msecs_to_jiffies(const ulong val) +{ + return msecs_to_jiffies(val); +} + +static int sysctl_u2k_int_conv_ms(const bool *negp, const ulong *u_ptr, int *k_ptr) +{ + return proc_int_u2k_conv_uop(u_ptr, k_ptr, negp, sysctl_msecs_to_jiffies); +} + +static ulong sysctl_jiffies_to_msecs(const ulong val) +{ + return jiffies_to_msecs(val); +} + +static int sysctl_k2u_int_conv_ms(bool *negp, ulong *u_ptr, const int *k_ptr) +{ + return proc_int_k2u_conv_kop(u_ptr, k_ptr, negp, sysctl_jiffies_to_msecs); +} + +static int do_proc_int_conv_jiffies(bool *negp, ulong *u_ptr, int *k_ptr, + int dir, const struct ctl_table *tbl) +{ + return proc_int_conv(negp, u_ptr, k_ptr, dir, tbl, false, + sysctl_u2k_int_conv_hz, sysctl_k2u_int_conv_hz); +} + +static int do_proc_int_conv_userhz_jiffies(bool *negp, ulong *u_ptr, + int *k_ptr, int dir, + const struct ctl_table *tbl) +{ + return proc_int_conv(negp, u_ptr, k_ptr, dir, tbl, false, + sysctl_u2k_int_conv_userhz, + sysctl_k2u_int_conv_userhz); +} + +static int do_proc_int_conv_ms_jiffies(bool *negp, ulong *u_ptr, int *k_ptr, + int dir, const struct ctl_table *tbl) +{ + return proc_int_conv(negp, u_ptr, k_ptr, dir, tbl, false, + sysctl_u2k_int_conv_ms, sysctl_k2u_int_conv_ms); +} + +static int do_proc_int_conv_ms_jiffies_minmax(bool *negp, ulong *u_ptr, + int *k_ptr, int dir, + const struct ctl_table *tbl) +{ + return proc_int_conv(negp, u_ptr, k_ptr, dir, tbl, false, + sysctl_u2k_int_conv_ms, sysctl_k2u_int_conv_ms); +} + +#else // CONFIG_PROC_SYSCTL +static int do_proc_int_conv_jiffies(bool *negp, ulong *u_ptr, int *k_ptr, + int dir, const struct ctl_table *tbl) +{ + return -ENOSYS; +} + +static int do_proc_int_conv_userhz_jiffies(bool *negp, ulong *u_ptr, + int *k_ptr, int dir, + const struct ctl_table *tbl) +{ + return -ENOSYS; +} + +static int do_proc_int_conv_ms_jiffies(bool *negp, ulong *u_ptr, int *k_ptr, + int dir, const struct ctl_table *tbl) +{ + return -ENOSYS; +} + +static int do_proc_int_conv_ms_jiffies_minmax(bool *negp, ulong *u_ptr, + int *k_ptr, int dir, + const struct ctl_table *tbl) +{ + return -ENOSYS; +} +#endif /** * proc_dointvec_jiffies - read a vector of integers as seconds diff --git a/net/ceph/Kconfig b/net/ceph/Kconfig index ea60e3ef0834..7e2528cde4b9 100644 --- a/net/ceph/Kconfig +++ b/net/ceph/Kconfig @@ -6,6 +6,7 @@ config CEPH_LIB select CRYPTO_AES select CRYPTO_CBC select CRYPTO_GCM + select CRYPTO_KRB5 select CRYPTO_LIB_SHA256 select CRYPTO select KEYS diff --git a/net/ceph/auth_x.c b/net/ceph/auth_x.c index a21c157daf7d..13b3df9af0ac 100644 --- a/net/ceph/auth_x.c +++ b/net/ceph/auth_x.c @@ -17,6 +17,22 @@ #include "auth_x.h" #include "auth_x_protocol.h" +static const u32 ticket_key_usages[] = { + CEPHX_KEY_USAGE_TICKET_SESSION_KEY, + CEPHX_KEY_USAGE_TICKET_BLOB, + CEPHX_KEY_USAGE_AUTH_CONNECTION_SECRET +}; + +static const u32 authorizer_key_usages[] = { + CEPHX_KEY_USAGE_AUTHORIZE, + CEPHX_KEY_USAGE_AUTHORIZE_CHALLENGE, + CEPHX_KEY_USAGE_AUTHORIZE_REPLY +}; + +static const u32 client_key_usages[] = { + CEPHX_KEY_USAGE_TICKET_SESSION_KEY +}; + static void ceph_x_validate_tickets(struct ceph_auth_client *ac, int *pneed); static int ceph_x_is_authenticated(struct ceph_auth_client *ac) @@ -44,28 +60,41 @@ static int ceph_x_should_authenticate(struct ceph_auth_client *ac) return !!need; } -static int ceph_x_encrypt_offset(void) +static int __ceph_x_encrypt_offset(const struct ceph_crypto_key *key) { - return sizeof(u32) + sizeof(struct ceph_x_encrypt_header); + return ceph_crypt_data_offset(key) + + sizeof(struct ceph_x_encrypt_header); } -static int ceph_x_encrypt_buflen(int ilen) +static int ceph_x_encrypt_offset(const struct ceph_crypto_key *key) { - return ceph_x_encrypt_offset() + ilen + 16; + return sizeof(u32) + __ceph_x_encrypt_offset(key); } -static int ceph_x_encrypt(struct ceph_crypto_key *secret, void *buf, - int buf_len, int plaintext_len) +/* + * AES: ciphertext_len | hdr | data... | padding + * AES256KRB5: ciphertext_len | confounder | hdr | data... | hmac + */ +static int ceph_x_encrypt_buflen(const struct ceph_crypto_key *key, + int data_len) { - struct ceph_x_encrypt_header *hdr = buf + sizeof(u32); + int encrypt_len = sizeof(struct ceph_x_encrypt_header) + data_len; + return sizeof(u32) + ceph_crypt_buflen(key, encrypt_len); +} + +static int ceph_x_encrypt(const struct ceph_crypto_key *key, int usage_slot, + void *buf, int buf_len, int plaintext_len) +{ + struct ceph_x_encrypt_header *hdr; int ciphertext_len; int ret; + hdr = buf + sizeof(u32) + ceph_crypt_data_offset(key); hdr->struct_v = 1; hdr->magic = cpu_to_le64(CEPHX_ENC_MAGIC); - ret = ceph_crypt(secret, true, buf + sizeof(u32), buf_len - sizeof(u32), - plaintext_len + sizeof(struct ceph_x_encrypt_header), + ret = ceph_crypt(key, usage_slot, true, buf + sizeof(u32), + buf_len - sizeof(u32), plaintext_len + sizeof(*hdr), &ciphertext_len); if (ret) return ret; @@ -74,18 +103,19 @@ static int ceph_x_encrypt(struct ceph_crypto_key *secret, void *buf, return sizeof(u32) + ciphertext_len; } -static int __ceph_x_decrypt(struct ceph_crypto_key *secret, void *p, - int ciphertext_len) +static int __ceph_x_decrypt(const struct ceph_crypto_key *key, int usage_slot, + void *p, int ciphertext_len) { - struct ceph_x_encrypt_header *hdr = p; + struct ceph_x_encrypt_header *hdr; int plaintext_len; int ret; - ret = ceph_crypt(secret, false, p, ciphertext_len, ciphertext_len, - &plaintext_len); + ret = ceph_crypt(key, usage_slot, false, p, ciphertext_len, + ciphertext_len, &plaintext_len); if (ret) return ret; + hdr = p + ceph_crypt_data_offset(key); if (le64_to_cpu(hdr->magic) != CEPHX_ENC_MAGIC) { pr_err("%s bad magic\n", __func__); return -EINVAL; @@ -94,7 +124,8 @@ static int __ceph_x_decrypt(struct ceph_crypto_key *secret, void *p, return plaintext_len - sizeof(*hdr); } -static int ceph_x_decrypt(struct ceph_crypto_key *secret, void **p, void *end) +static int ceph_x_decrypt(const struct ceph_crypto_key *key, int usage_slot, + void **p, void *end) { int ciphertext_len; int ret; @@ -102,7 +133,7 @@ static int ceph_x_decrypt(struct ceph_crypto_key *secret, void **p, void *end) ceph_decode_32_safe(p, end, ciphertext_len, e_inval); ceph_decode_need(p, end, ciphertext_len, e_inval); - ret = __ceph_x_decrypt(secret, *p, ciphertext_len); + ret = __ceph_x_decrypt(key, usage_slot, *p, ciphertext_len); if (ret < 0) return ret; @@ -193,8 +224,10 @@ static int process_one_ticket(struct ceph_auth_client *ac, } /* blob for me */ - dp = *p + ceph_x_encrypt_offset(); - ret = ceph_x_decrypt(secret, p, end); + dp = *p + ceph_x_encrypt_offset(secret); + ret = ceph_x_decrypt(secret, + 0 /* CEPHX_KEY_USAGE_TICKET_SESSION_KEY */, + p, end); if (ret < 0) goto out; dout(" decrypted %d bytes\n", ret); @@ -208,6 +241,11 @@ static int process_one_ticket(struct ceph_auth_client *ac, if (ret) goto out; + ret = ceph_crypto_key_prepare(&new_session_key, ticket_key_usages, + ARRAY_SIZE(ticket_key_usages)); + if (ret) + goto out; + ceph_decode_need(&dp, dend, sizeof(struct ceph_timespec), bad); ceph_decode_timespec64(&validity, dp); dp += sizeof(struct ceph_timespec); @@ -220,8 +258,10 @@ static int process_one_ticket(struct ceph_auth_client *ac, ceph_decode_8_safe(p, end, is_enc, bad); if (is_enc) { /* encrypted */ - tp = *p + ceph_x_encrypt_offset(); - ret = ceph_x_decrypt(&th->session_key, p, end); + tp = *p + ceph_x_encrypt_offset(&th->session_key); + ret = ceph_x_decrypt(&th->session_key, + 1 /* CEPHX_KEY_USAGE_TICKET_BLOB */, + p, end); if (ret < 0) goto out; dout(" encrypted ticket, decrypted %d bytes\n", ret); @@ -312,7 +352,7 @@ static int encrypt_authorizer(struct ceph_x_authorizer *au, p = (void *)(msg_a + 1) + le32_to_cpu(msg_a->ticket_blob.blob_len); end = au->buf->vec.iov_base + au->buf->vec.iov_len; - msg_b = p + ceph_x_encrypt_offset(); + msg_b = p + ceph_x_encrypt_offset(&au->session_key); msg_b->struct_v = 2; msg_b->nonce = cpu_to_le64(au->nonce); if (server_challenge) { @@ -324,7 +364,9 @@ static int encrypt_authorizer(struct ceph_x_authorizer *au, msg_b->server_challenge_plus_one = 0; } - ret = ceph_x_encrypt(&au->session_key, p, end - p, sizeof(*msg_b)); + ret = ceph_x_encrypt(&au->session_key, + 0 /* CEPHX_KEY_USAGE_AUTHORIZE */, + p, end - p, sizeof(*msg_b)); if (ret < 0) return ret; @@ -367,8 +409,13 @@ static int ceph_x_build_authorizer(struct ceph_auth_client *ac, if (ret) goto out_au; + ret = ceph_crypto_key_prepare(&au->session_key, authorizer_key_usages, + ARRAY_SIZE(authorizer_key_usages)); + if (ret) + goto out_au; + maxlen = sizeof(*msg_a) + ticket_blob_len + - ceph_x_encrypt_buflen(sizeof(*msg_b)); + ceph_x_encrypt_buflen(&au->session_key, sizeof(*msg_b)); dout(" need len %d\n", maxlen); if (au->buf && au->buf->alloc_len < maxlen) { ceph_buffer_put(au->buf); @@ -506,8 +553,7 @@ static int ceph_x_build_request(struct ceph_auth_client *ac, if (need & CEPH_ENTITY_TYPE_AUTH) { struct ceph_x_authenticate *auth = (void *)(head + 1); void *enc_buf = xi->auth_authorizer.enc_buf; - struct ceph_x_challenge_blob *blob = enc_buf + - ceph_x_encrypt_offset(); + struct ceph_x_challenge_blob *blob; u64 *u; p = auth + 1; @@ -517,14 +563,29 @@ static int ceph_x_build_request(struct ceph_auth_client *ac, dout(" get_auth_session_key\n"); head->op = cpu_to_le16(CEPHX_GET_AUTH_SESSION_KEY); - /* encrypt and hash */ + if (xi->secret.type == CEPH_CRYPTO_AES) { + blob = enc_buf + ceph_x_encrypt_offset(&xi->secret); + } else { + BUILD_BUG_ON(SHA256_DIGEST_SIZE + sizeof(*blob) > + CEPHX_AU_ENC_BUF_LEN); + blob = enc_buf + SHA256_DIGEST_SIZE; + } + get_random_bytes(&auth->client_challenge, sizeof(u64)); blob->client_challenge = auth->client_challenge; blob->server_challenge = cpu_to_le64(xi->server_challenge); - ret = ceph_x_encrypt(&xi->secret, enc_buf, CEPHX_AU_ENC_BUF_LEN, - sizeof(*blob)); - if (ret < 0) - return ret; + + if (xi->secret.type == CEPH_CRYPTO_AES) { + ret = ceph_x_encrypt(&xi->secret, 0 /* dummy */, + enc_buf, CEPHX_AU_ENC_BUF_LEN, + sizeof(*blob)); + if (ret < 0) + return ret; + } else { + ceph_hmac_sha256(&xi->secret, blob, sizeof(*blob), + enc_buf); + ret = SHA256_DIGEST_SIZE; + } auth->struct_v = 3; /* nautilus+ */ auth->key = 0; @@ -634,8 +695,10 @@ static int handle_auth_session_key(struct ceph_auth_client *ac, u64 global_id, ceph_decode_need(p, end, len, e_inval); dout("%s connection secret blob len %d\n", __func__, len); if (len > 0) { - dp = *p + ceph_x_encrypt_offset(); - ret = ceph_x_decrypt(&th->session_key, p, *p + len); + dp = *p + ceph_x_encrypt_offset(&th->session_key); + ret = ceph_x_decrypt(&th->session_key, + 2 /* CEPHX_KEY_USAGE_AUTH_CONNECTION_SECRET */, + p, *p + len); if (ret < 0) return ret; @@ -799,12 +862,14 @@ static int decrypt_authorizer_challenge(struct ceph_crypto_key *secret, int ret; /* no leading len */ - ret = __ceph_x_decrypt(secret, challenge, challenge_len); + ret = __ceph_x_decrypt(secret, + 1 /* CEPHX_KEY_USAGE_AUTHORIZE_CHALLENGE */, + challenge, challenge_len); if (ret < 0) return ret; dout("%s decrypted %d bytes\n", __func__, ret); - dp = challenge + sizeof(struct ceph_x_encrypt_header); + dp = challenge + __ceph_x_encrypt_offset(secret); dend = dp + ret; ceph_decode_skip_8(&dp, dend, e_inval); /* struct_v */ @@ -851,8 +916,9 @@ static int decrypt_authorizer_reply(struct ceph_crypto_key *secret, u8 struct_v; int ret; - dp = *p + ceph_x_encrypt_offset(); - ret = ceph_x_decrypt(secret, p, end); + dp = *p + ceph_x_encrypt_offset(secret); + ret = ceph_x_decrypt(secret, 2 /* CEPHX_KEY_USAGE_AUTHORIZE_REPLY */, + p, end); if (ret < 0) return ret; @@ -974,7 +1040,8 @@ static int calc_signature(struct ceph_x_authorizer *au, struct ceph_msg *msg, __le32 front_crc; __le32 middle_crc; __le32 data_crc; - } __packed *sigblock = enc_buf + ceph_x_encrypt_offset(); + } __packed *sigblock = enc_buf + + ceph_x_encrypt_offset(&au->session_key); sigblock->len = cpu_to_le32(4*sizeof(u32)); sigblock->header_crc = msg->hdr.crc; @@ -982,8 +1049,9 @@ static int calc_signature(struct ceph_x_authorizer *au, struct ceph_msg *msg, sigblock->middle_crc = msg->footer.middle_crc; sigblock->data_crc = msg->footer.data_crc; - ret = ceph_x_encrypt(&au->session_key, enc_buf, - CEPHX_AU_ENC_BUF_LEN, sizeof(*sigblock)); + ret = ceph_x_encrypt(&au->session_key, 0 /* dummy */, + enc_buf, CEPHX_AU_ENC_BUF_LEN, + sizeof(*sigblock)); if (ret < 0) return ret; @@ -998,11 +1066,19 @@ static int calc_signature(struct ceph_x_authorizer *au, struct ceph_msg *msg, __le32 data_crc; __le32 data_len; __le32 seq_lower_word; - } __packed *sigblock = enc_buf; + } __packed *sigblock; struct { __le64 a, b, c, d; } __packed *penc = enc_buf; - int ciphertext_len; + + if (au->session_key.type == CEPH_CRYPTO_AES) { + /* no leading len, no ceph_x_encrypt_header */ + sigblock = enc_buf; + } else { + BUILD_BUG_ON(SHA256_DIGEST_SIZE + sizeof(*sigblock) > + CEPHX_AU_ENC_BUF_LEN); + sigblock = enc_buf + SHA256_DIGEST_SIZE; + } sigblock->header_crc = msg->hdr.crc; sigblock->front_crc = msg->footer.front_crc; @@ -1013,12 +1089,18 @@ static int calc_signature(struct ceph_x_authorizer *au, struct ceph_msg *msg, sigblock->data_len = msg->hdr.data_len; sigblock->seq_lower_word = *(__le32 *)&msg->hdr.seq; - /* no leading len, no ceph_x_encrypt_header */ - ret = ceph_crypt(&au->session_key, true, enc_buf, - CEPHX_AU_ENC_BUF_LEN, sizeof(*sigblock), - &ciphertext_len); - if (ret) - return ret; + if (au->session_key.type == CEPH_CRYPTO_AES) { + int ciphertext_len; /* unused */ + + ret = ceph_crypt(&au->session_key, 0 /* dummy */, + true, enc_buf, CEPHX_AU_ENC_BUF_LEN, + sizeof(*sigblock), &ciphertext_len); + if (ret) + return ret; + } else { + ceph_hmac_sha256(&au->session_key, sigblock, + sizeof(*sigblock), enc_buf); + } *psig = penc->a ^ penc->b ^ penc->c ^ penc->d; } @@ -1092,21 +1174,27 @@ int ceph_x_init(struct ceph_auth_client *ac) int ret; dout("ceph_x_init %p\n", ac); - ret = -ENOMEM; xi = kzalloc(sizeof(*xi), GFP_NOFS); if (!xi) - goto out; + return -ENOMEM; ret = -EINVAL; if (!ac->key) { pr_err("no secret set (for auth_x protocol)\n"); - goto out_nomem; + goto err_xi; } ret = ceph_crypto_key_clone(&xi->secret, ac->key); if (ret < 0) { pr_err("cannot clone key: %d\n", ret); - goto out_nomem; + goto err_xi; + } + + ret = ceph_crypto_key_prepare(&xi->secret, client_key_usages, + ARRAY_SIZE(client_key_usages)); + if (ret) { + pr_err("cannot prepare key: %d\n", ret); + goto err_secret; } xi->starting = true; @@ -1117,8 +1205,9 @@ int ceph_x_init(struct ceph_auth_client *ac) ac->ops = &ceph_x_ops; return 0; -out_nomem: +err_secret: + ceph_crypto_key_destroy(&xi->secret); +err_xi: kfree(xi); -out: return ret; } diff --git a/net/ceph/auth_x_protocol.h b/net/ceph/auth_x_protocol.h index 9c60feeb1bcb..d097b3651c99 100644 --- a/net/ceph/auth_x_protocol.h +++ b/net/ceph/auth_x_protocol.h @@ -6,6 +6,44 @@ #define CEPHX_GET_PRINCIPAL_SESSION_KEY 0x0200 #define CEPHX_GET_ROTATING_KEY 0x0400 +/* Client <-> AuthMonitor */ +/* + * The AUTH session's connection secret: encrypted with the AUTH + * ticket session key + */ +#define CEPHX_KEY_USAGE_AUTH_CONNECTION_SECRET 0x03 +/* + * The ticket's blob for the client ("blob for me", contains the + * session key): encrypted with the client's secret key in case of + * the AUTH ticket and the AUTH ticket session key in case of other + * service tickets + */ +#define CEPHX_KEY_USAGE_TICKET_SESSION_KEY 0x04 +/* + * The ticket's blob for the service (ceph_x_ticket_blob): possibly + * encrypted with the old AUTH ticket session key in case of the AUTH + * ticket and not encrypted in case of other service tickets + */ +#define CEPHX_KEY_USAGE_TICKET_BLOB 0x05 + +/* Client <-> Service */ +/* + * The client's authorization request (ceph_x_authorize_b): + * encrypted with the service ticket session key + */ +#define CEPHX_KEY_USAGE_AUTHORIZE 0x10 +/* + * The service's challenge (ceph_x_authorize_challenge): + * encrypted with the service ticket session key + */ +#define CEPHX_KEY_USAGE_AUTHORIZE_CHALLENGE 0x11 +/* + * The service's final reply (ceph_x_authorize_reply + the service + * session's connection secret): encrypted with the service ticket + * session key + */ +#define CEPHX_KEY_USAGE_AUTHORIZE_REPLY 0x12 + /* common bits */ struct ceph_x_ticket_blob { __u8 struct_v; diff --git a/net/ceph/crypto.c b/net/ceph/crypto.c index 01b2ce1e8fc0..b2067ea6c38a 100644 --- a/net/ceph/crypto.c +++ b/net/ceph/crypto.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -16,77 +17,119 @@ #include #include "crypto.h" -/* - * Set ->key and ->tfm. The rest of the key should be filled in before - * this function is called. - */ -static int set_secret(struct ceph_crypto_key *key, void *buf) +static int set_aes_tfm(struct ceph_crypto_key *key) { unsigned int noio_flag; int ret; - key->key = NULL; - key->tfm = NULL; + noio_flag = memalloc_noio_save(); + key->aes_tfm = crypto_alloc_sync_skcipher("cbc(aes)", 0, 0); + memalloc_noio_restore(noio_flag); + if (IS_ERR(key->aes_tfm)) { + ret = PTR_ERR(key->aes_tfm); + key->aes_tfm = NULL; + return ret; + } + ret = crypto_sync_skcipher_setkey(key->aes_tfm, key->key, key->len); + if (ret) + return ret; + + return 0; +} + +static int set_krb5_tfms(struct ceph_crypto_key *key, const u32 *key_usages, + int key_usage_cnt) +{ + struct krb5_buffer TK = { .len = key->len, .data = key->key }; + unsigned int noio_flag; + int ret = 0; + int i; + + if (WARN_ON_ONCE(key_usage_cnt > ARRAY_SIZE(key->krb5_tfms))) + return -EINVAL; + + key->krb5_type = crypto_krb5_find_enctype( + KRB5_ENCTYPE_AES256_CTS_HMAC_SHA384_192); + if (!key->krb5_type) + return -ENOPKG; + + /* + * Despite crypto_krb5_prepare_encryption() taking a gfp mask, + * crypto_alloc_aead() inside of it allocates with GFP_KERNEL. + */ + noio_flag = memalloc_noio_save(); + for (i = 0; i < key_usage_cnt; i++) { + key->krb5_tfms[i] = crypto_krb5_prepare_encryption( + key->krb5_type, &TK, key_usages[i], + GFP_NOIO); + if (IS_ERR(key->krb5_tfms[i])) { + ret = PTR_ERR(key->krb5_tfms[i]); + key->krb5_tfms[i] = NULL; + goto out_flag; + } + } + +out_flag: + memalloc_noio_restore(noio_flag); + return ret; +} + +int ceph_crypto_key_prepare(struct ceph_crypto_key *key, + const u32 *key_usages, int key_usage_cnt) +{ switch (key->type) { case CEPH_CRYPTO_NONE: return 0; /* nothing to do */ case CEPH_CRYPTO_AES: - break; + return set_aes_tfm(key); + case CEPH_CRYPTO_AES256KRB5: + hmac_sha256_preparekey(&key->hmac_key, key->key, key->len); + return set_krb5_tfms(key, key_usages, key_usage_cnt); default: return -ENOTSUPP; } - - if (!key->len) - return -EINVAL; - - key->key = kmemdup(buf, key->len, GFP_NOIO); - if (!key->key) { - ret = -ENOMEM; - goto fail; - } - - /* crypto_alloc_sync_skcipher() allocates with GFP_KERNEL */ - noio_flag = memalloc_noio_save(); - key->tfm = crypto_alloc_sync_skcipher("cbc(aes)", 0, 0); - memalloc_noio_restore(noio_flag); - if (IS_ERR(key->tfm)) { - ret = PTR_ERR(key->tfm); - key->tfm = NULL; - goto fail; - } - - ret = crypto_sync_skcipher_setkey(key->tfm, key->key, key->len); - if (ret) - goto fail; - - return 0; - -fail: - ceph_crypto_key_destroy(key); - return ret; } +/* + * @dst should be zeroed before this function is called. + */ int ceph_crypto_key_clone(struct ceph_crypto_key *dst, const struct ceph_crypto_key *src) { - memcpy(dst, src, sizeof(struct ceph_crypto_key)); - return set_secret(dst, src->key); + dst->type = src->type; + dst->created = src->created; + dst->len = src->len; + + dst->key = kmemdup(src->key, src->len, GFP_NOIO); + if (!dst->key) + return -ENOMEM; + + return 0; } +/* + * @key should be zeroed before this function is called. + */ int ceph_crypto_key_decode(struct ceph_crypto_key *key, void **p, void *end) { - int ret; - ceph_decode_need(p, end, 2*sizeof(u16) + sizeof(key->created), bad); key->type = ceph_decode_16(p); ceph_decode_copy(p, &key->created, sizeof(key->created)); key->len = ceph_decode_16(p); ceph_decode_need(p, end, key->len, bad); - ret = set_secret(key, *p); + if (key->len > CEPH_MAX_KEY_LEN) { + pr_err("secret too big %d\n", key->len); + return -EINVAL; + } + + key->key = kmemdup(*p, key->len, GFP_NOIO); + if (!key->key) + return -ENOMEM; + memzero_explicit(*p, key->len); *p += key->len; - return ret; + return 0; bad: dout("failed to decode crypto key\n"); @@ -122,12 +165,26 @@ int ceph_crypto_key_unarmor(struct ceph_crypto_key *key, const char *inkey) void ceph_crypto_key_destroy(struct ceph_crypto_key *key) { - if (key) { - kfree_sensitive(key->key); - key->key = NULL; - if (key->tfm) { - crypto_free_sync_skcipher(key->tfm); - key->tfm = NULL; + int i; + + if (!key) + return; + + kfree_sensitive(key->key); + key->key = NULL; + + if (key->type == CEPH_CRYPTO_AES) { + if (key->aes_tfm) { + crypto_free_sync_skcipher(key->aes_tfm); + key->aes_tfm = NULL; + } + } else if (key->type == CEPH_CRYPTO_AES256KRB5) { + memzero_explicit(&key->hmac_key, sizeof(key->hmac_key)); + for (i = 0; i < ARRAY_SIZE(key->krb5_tfms); i++) { + if (key->krb5_tfms[i]) { + crypto_free_aead(key->krb5_tfms[i]); + key->krb5_tfms[i] = NULL; + } } } } @@ -207,7 +264,7 @@ static void teardown_sgtable(struct sg_table *sgt) static int ceph_aes_crypt(const struct ceph_crypto_key *key, bool encrypt, void *buf, int buf_len, int in_len, int *pout_len) { - SYNC_SKCIPHER_REQUEST_ON_STACK(req, key->tfm); + SYNC_SKCIPHER_REQUEST_ON_STACK(req, key->aes_tfm); struct sg_table sgt; struct scatterlist prealloc_sg; char iv[AES_BLOCK_SIZE] __aligned(8); @@ -223,7 +280,7 @@ static int ceph_aes_crypt(const struct ceph_crypto_key *key, bool encrypt, return ret; memcpy(iv, aes_iv, AES_BLOCK_SIZE); - skcipher_request_set_sync_tfm(req, key->tfm); + skcipher_request_set_sync_tfm(req, key->aes_tfm); skcipher_request_set_callback(req, 0, NULL, NULL); skcipher_request_set_crypt(req, sgt.sgl, sgt.sgl, crypt_len, iv); @@ -268,7 +325,68 @@ out_sgt: return ret; } -int ceph_crypt(const struct ceph_crypto_key *key, bool encrypt, +static int ceph_krb5_encrypt(const struct ceph_crypto_key *key, int usage_slot, + void *buf, int buf_len, int in_len, int *pout_len) +{ + struct sg_table sgt; + struct scatterlist prealloc_sg; + int ret; + + if (WARN_ON_ONCE(usage_slot >= ARRAY_SIZE(key->krb5_tfms))) + return -EINVAL; + + ret = setup_sgtable(&sgt, &prealloc_sg, buf, buf_len); + if (ret) + return ret; + + ret = crypto_krb5_encrypt(key->krb5_type, key->krb5_tfms[usage_slot], + sgt.sgl, sgt.nents, buf_len, AES_BLOCK_SIZE, + in_len, false); + if (ret < 0) { + pr_err("%s encrypt failed: %d\n", __func__, ret); + goto out_sgt; + } + + *pout_len = ret; + ret = 0; + +out_sgt: + teardown_sgtable(&sgt); + return ret; +} + +static int ceph_krb5_decrypt(const struct ceph_crypto_key *key, int usage_slot, + void *buf, int buf_len, int in_len, int *pout_len) +{ + struct sg_table sgt; + struct scatterlist prealloc_sg; + size_t data_off = 0; + size_t data_len = in_len; + int ret; + + if (WARN_ON_ONCE(usage_slot >= ARRAY_SIZE(key->krb5_tfms))) + return -EINVAL; + + ret = setup_sgtable(&sgt, &prealloc_sg, buf, in_len); + if (ret) + return ret; + + ret = crypto_krb5_decrypt(key->krb5_type, key->krb5_tfms[usage_slot], + sgt.sgl, sgt.nents, &data_off, &data_len); + if (ret) { + pr_err("%s decrypt failed: %d\n", __func__, ret); + goto out_sgt; + } + + WARN_ON(data_off != AES_BLOCK_SIZE); + *pout_len = data_len; + +out_sgt: + teardown_sgtable(&sgt); + return ret; +} + +int ceph_crypt(const struct ceph_crypto_key *key, int usage_slot, bool encrypt, void *buf, int buf_len, int in_len, int *pout_len) { switch (key->type) { @@ -278,11 +396,64 @@ int ceph_crypt(const struct ceph_crypto_key *key, bool encrypt, case CEPH_CRYPTO_AES: return ceph_aes_crypt(key, encrypt, buf, buf_len, in_len, pout_len); + case CEPH_CRYPTO_AES256KRB5: + return encrypt ? + ceph_krb5_encrypt(key, usage_slot, buf, buf_len, in_len, + pout_len) : + ceph_krb5_decrypt(key, usage_slot, buf, buf_len, in_len, + pout_len); default: return -ENOTSUPP; } } +int ceph_crypt_data_offset(const struct ceph_crypto_key *key) +{ + switch (key->type) { + case CEPH_CRYPTO_NONE: + case CEPH_CRYPTO_AES: + return 0; + case CEPH_CRYPTO_AES256KRB5: + /* confounder */ + return AES_BLOCK_SIZE; + default: + BUG(); + } +} + +int ceph_crypt_buflen(const struct ceph_crypto_key *key, int data_len) +{ + switch (key->type) { + case CEPH_CRYPTO_NONE: + return data_len; + case CEPH_CRYPTO_AES: + /* PKCS#7 padding at the end */ + return data_len + AES_BLOCK_SIZE - + (data_len & (AES_BLOCK_SIZE - 1)); + case CEPH_CRYPTO_AES256KRB5: + /* confounder at the beginning and 192-bit HMAC at the end */ + return AES_BLOCK_SIZE + data_len + 24; + default: + BUG(); + } +} + +void ceph_hmac_sha256(const struct ceph_crypto_key *key, const void *buf, + int buf_len, u8 hmac[SHA256_DIGEST_SIZE]) +{ + switch (key->type) { + case CEPH_CRYPTO_NONE: + case CEPH_CRYPTO_AES: + memset(hmac, 0, SHA256_DIGEST_SIZE); + return; + case CEPH_CRYPTO_AES256KRB5: + hmac_sha256(&key->hmac_key, buf, buf_len, hmac); + return; + default: + BUG(); + } +} + static int ceph_key_preparse(struct key_preparsed_payload *prep) { struct ceph_crypto_key *ckey; @@ -295,7 +466,7 @@ static int ceph_key_preparse(struct key_preparsed_payload *prep) goto err; ret = -ENOMEM; - ckey = kmalloc(sizeof(*ckey), GFP_KERNEL); + ckey = kzalloc(sizeof(*ckey), GFP_KERNEL); if (!ckey) goto err; diff --git a/net/ceph/crypto.h b/net/ceph/crypto.h index 23de29fc613c..3a2ade15abbc 100644 --- a/net/ceph/crypto.h +++ b/net/ceph/crypto.h @@ -2,10 +2,11 @@ #ifndef _FS_CEPH_CRYPTO_H #define _FS_CEPH_CRYPTO_H +#include #include #include -#define CEPH_KEY_LEN 16 +#define CEPH_MAX_KEY_LEN 32 #define CEPH_MAX_CON_SECRET_LEN 64 /* @@ -16,9 +17,19 @@ struct ceph_crypto_key { struct ceph_timespec created; int len; void *key; - struct crypto_sync_skcipher *tfm; + + union { + struct crypto_sync_skcipher *aes_tfm; + struct { + struct hmac_sha256_key hmac_key; + const struct krb5_enctype *krb5_type; + struct crypto_aead *krb5_tfms[3]; + }; + }; }; +int ceph_crypto_key_prepare(struct ceph_crypto_key *key, + const u32 *key_usages, int key_usage_cnt); int ceph_crypto_key_clone(struct ceph_crypto_key *dst, const struct ceph_crypto_key *src); int ceph_crypto_key_decode(struct ceph_crypto_key *key, void **p, void *end); @@ -26,8 +37,12 @@ int ceph_crypto_key_unarmor(struct ceph_crypto_key *key, const char *in); void ceph_crypto_key_destroy(struct ceph_crypto_key *key); /* crypto.c */ -int ceph_crypt(const struct ceph_crypto_key *key, bool encrypt, +int ceph_crypt(const struct ceph_crypto_key *key, int usage_slot, bool encrypt, void *buf, int buf_len, int in_len, int *pout_len); +int ceph_crypt_data_offset(const struct ceph_crypto_key *key); +int ceph_crypt_buflen(const struct ceph_crypto_key *key, int data_len); +void ceph_hmac_sha256(const struct ceph_crypto_key *key, const void *buf, + int buf_len, u8 hmac[SHA256_DIGEST_SIZE]); int ceph_crypto_init(void); void ceph_crypto_shutdown(void); diff --git a/net/ceph/messenger_v2.c b/net/ceph/messenger_v2.c index c9d50c0dcd33..5ec3272cd2dd 100644 --- a/net/ceph/messenger_v2.c +++ b/net/ceph/messenger_v2.c @@ -779,9 +779,9 @@ static int setup_crypto(struct ceph_connection *con, return 0; /* auth_x, secure mode */ } -static void ceph_hmac_sha256(struct ceph_connection *con, - const struct kvec *kvecs, int kvec_cnt, - u8 hmac[SHA256_DIGEST_SIZE]) +static void con_hmac_sha256(struct ceph_connection *con, + const struct kvec *kvecs, int kvec_cnt, + u8 hmac[SHA256_DIGEST_SIZE]) { struct hmac_sha256_ctx ctx; int i; @@ -1438,8 +1438,8 @@ static int prepare_auth_signature(struct ceph_connection *con) if (!buf) return -ENOMEM; - ceph_hmac_sha256(con, con->v2.in_sign_kvecs, con->v2.in_sign_kvec_cnt, - CTRL_BODY(buf)); + con_hmac_sha256(con, con->v2.in_sign_kvecs, con->v2.in_sign_kvec_cnt, + CTRL_BODY(buf)); return prepare_control(con, FRAME_TAG_AUTH_SIGNATURE, buf, SHA256_DIGEST_SIZE); @@ -2360,7 +2360,7 @@ bad: */ static int process_auth_done(struct ceph_connection *con, void *p, void *end) { - u8 session_key_buf[CEPH_KEY_LEN + 16]; + u8 session_key_buf[CEPH_MAX_KEY_LEN + 16]; u8 con_secret_buf[CEPH_MAX_CON_SECRET_LEN + 16]; u8 *session_key = PTR_ALIGN(&session_key_buf[0], 16); u8 *con_secret = PTR_ALIGN(&con_secret_buf[0], 16); @@ -2436,8 +2436,8 @@ static int process_auth_signature(struct ceph_connection *con, return -EINVAL; } - ceph_hmac_sha256(con, con->v2.out_sign_kvecs, con->v2.out_sign_kvec_cnt, - hmac); + con_hmac_sha256(con, con->v2.out_sign_kvecs, con->v2.out_sign_kvec_cnt, + hmac); ceph_decode_need(&p, end, SHA256_DIGEST_SIZE, bad); if (crypto_memneq(p, hmac, SHA256_DIGEST_SIZE)) { diff --git a/scripts/cc-can-link.sh b/scripts/cc-can-link.sh index e67fd8d7b684..58dc7dd6d556 100755 --- a/scripts/cc-can-link.sh +++ b/scripts/cc-can-link.sh @@ -5,7 +5,7 @@ cat << "END" | $@ -Werror -Wl,--fatal-warnings -x c - -o /dev/null >/dev/null 2> #include int main(void) { - printf(""); + printf("\n"); return 0; } END diff --git a/scripts/make_fit.py b/scripts/make_fit.py index e923cc8b05b7..15ba26974fd7 100755 --- a/scripts/make_fit.py +++ b/scripts/make_fit.py @@ -54,7 +54,7 @@ COMP_TOOLS = { 'bzip2': CompTool('.bz2', 'pbzip2,bzip2'), 'gzip': CompTool('.gz', 'pigz,gzip'), 'lz4': CompTool('.lz4', 'lz4'), - 'lzma': CompTool('.lzma', 'plzip,lzma'), + 'lzma': CompTool('.lzma', 'lzma'), 'lzo': CompTool('.lzo', 'lzop'), 'xz': CompTool('.xz', 'xz'), 'zstd': CompTool('.zstd', 'zstd'), diff --git a/scripts/package/kernel.spec b/scripts/package/kernel.spec index 0f1c8de1bd95..b3c956205af0 100644 --- a/scripts/package/kernel.spec +++ b/scripts/package/kernel.spec @@ -2,8 +2,6 @@ %{!?_arch: %define _arch dummy} %{!?make: %define make make} %define makeflags %{?_smp_mflags} ARCH=%{ARCH} -%define __spec_install_post /usr/lib/rpm/brp-compress || : -%define debug_package %{nil} Name: kernel Summary: The Linux Kernel @@ -47,14 +45,50 @@ This package provides kernel headers and makefiles sufficient to build modules against the %{version} kernel package. %endif -%if %{with_debuginfo} +%if %{with_debuginfo_manual} %package debuginfo Summary: Debug information package for the Linux kernel +Group: Development/Debug +AutoReq: 0 +AutoProv: 1 %description debuginfo This package provides debug information for the kernel image and modules from the %{version} package. +%define install_mod_strip 1 %endif +%if %{with_debuginfo_rpm} +# list of debuginfo-related options taken from distribution kernel.spec +# files +%undefine _include_minidebuginfo +%undefine _find_debuginfo_dwz_opts +%undefine _unique_build_ids +%undefine _unique_debug_names +%undefine _unique_debug_srcs +%undefine _debugsource_packages +%undefine _debuginfo_subpackages +%global _find_debuginfo_opts -r +%global _missing_build_ids_terminate_build 1 +%global _no_recompute_build_ids 1 +%{debug_package} + +# later, we make all modules executable so that find-debuginfo.sh strips +# them up. but they don't actually need to be executable, so remove the +# executable bit, taking care to do it _after_ find-debuginfo.sh has run +%define __spec_install_post \ + %{?__debug_package:%{__debug_install_post}} \ + %{__arch_install_post} \ + %{__os_install_post} \ + find %{buildroot}/lib/modules/%{KERNELRELEASE} -name "*.ko" -type f \\\ + | xargs --no-run-if-empty chmod u-x +%else +%define __spec_install_post /usr/lib/rpm/brp-compress || : +%endif +# some (but not all) versions of rpmbuild emit %%debug_package with +# %%install. since we've already emitted it manually, that would cause +# a package redefinition error. ensure that doesn't happen +%define debug_package %{nil} + %prep %setup -q -n linux cp %{SOURCE1} .config @@ -67,7 +101,7 @@ patch -p1 < %{SOURCE2} mkdir -p %{buildroot}/lib/modules/%{KERNELRELEASE} cp $(%{make} %{makeflags} -s image_name) %{buildroot}/lib/modules/%{KERNELRELEASE}/vmlinuz # DEPMOD=true makes depmod no-op. We do not package depmod-generated files. -%{make} %{makeflags} INSTALL_MOD_PATH=%{buildroot} INSTALL_MOD_STRIP=1 DEPMOD=true modules_install +%{make} %{makeflags} INSTALL_MOD_PATH=%{buildroot} %{?install_mod_strip:INSTALL_MOD_STRIP=1} DEPMOD=true modules_install %{make} %{makeflags} INSTALL_HDR_PATH=%{buildroot}/usr headers_install cp System.map %{buildroot}/lib/modules/%{KERNELRELEASE} cp .config %{buildroot}/lib/modules/%{KERNELRELEASE}/config @@ -98,22 +132,30 @@ ln -fns /usr/src/kernels/%{KERNELRELEASE} %{buildroot}/lib/modules/%{KERNELRELEA echo "%exclude /lib/modules/%{KERNELRELEASE}/build" } > %{buildroot}/kernel.list -%if %{with_debuginfo} +%if 0%{with_debuginfo_manual}%{with_debuginfo_rpm} > 0 # copying vmlinux directly to the debug directory means it will not get # stripped (but its source paths will still be collected + fixed up) mkdir -p %{buildroot}/usr/lib/debug/lib/modules/%{KERNELRELEASE} cp vmlinux %{buildroot}/usr/lib/debug/lib/modules/%{KERNELRELEASE} +%endif +%if %{with_debuginfo_rpm} +# make modules executable so that find-debuginfo.sh strips them. this +# will be undone later in %%__spec_install_post +find %{buildroot}/lib/modules/%{KERNELRELEASE} -name "*.ko" -type f \ + | xargs --no-run-if-empty chmod u+x +%endif + +%if %{with_debuginfo_manual} echo /usr/lib/debug/lib/modules/%{KERNELRELEASE}/vmlinux > %{buildroot}/debuginfo.list - while read -r mod; do mod="${mod%.o}.ko" dbg="%{buildroot}/usr/lib/debug/lib/modules/%{KERNELRELEASE}/kernel/${mod}" - buildid=$("${READELF}" -n "${mod}" | sed -n 's@^.*Build ID: \(..\)\(.*\)@\1/\2@p') + buildid=$("${READELF:-readelf}" -n "${mod}" | sed -n 's@^.*Build ID: \(..\)\(.*\)@\1/\2@p') link="%{buildroot}/usr/lib/debug/.build-id/${buildid}.debug" mkdir -p "${dbg%/*}" "${link%/*}" - "${OBJCOPY}" --only-keep-debug "${mod}" "${dbg}" + "${OBJCOPY:-objcopy}" --only-keep-debug "${mod}" "${dbg}" ln -sf --relative "${dbg}" "${link}" echo "${dbg#%{buildroot}}" >> %{buildroot}/debuginfo.list @@ -123,6 +165,10 @@ done < modules.order %clean rm -rf %{buildroot} +%if %{with_debuginfo_rpm} +rm -f debugfiles.list debuglinks.list debugsourcefiles.list debugsources.list \ + elfbins.list +%endif %post if [ -x /usr/bin/kernel-install ]; then @@ -162,7 +208,7 @@ fi /lib/modules/%{KERNELRELEASE}/build %endif -%if %{with_debuginfo} +%if %{with_debuginfo_manual} %files -f %{buildroot}/debuginfo.list debuginfo %defattr (-, root, root) %exclude /debuginfo.list diff --git a/scripts/package/mkspec b/scripts/package/mkspec index c7375bfc25a9..c604f8c174e2 100755 --- a/scripts/package/mkspec +++ b/scripts/package/mkspec @@ -23,15 +23,47 @@ else echo '%define with_devel 0' fi +# use %{debug_package} machinery to generate -debuginfo +with_debuginfo_rpm=0 +# manually generate -debuginfo package +with_debuginfo_manual=0 # debuginfo package generation uses find-debuginfo.sh under the hood, # which only works on uncompressed modules that contain debuginfo if grep -q CONFIG_DEBUG_INFO=y include/config/auto.conf && (! grep -q CONFIG_MODULE_COMPRESS=y include/config/auto.conf) && (! grep -q CONFIG_DEBUG_INFO_SPLIT=y include/config/auto.conf); then -echo '%define with_debuginfo %{?_without_debuginfo: 0} %{?!_without_debuginfo: 1}' -else -echo '%define with_debuginfo 0' + # If module signing is enabled (which may be required to boot with + # lockdown enabled), the find-debuginfo.sh machinery cannot be used + # because the signatures will be stripped off the modules. However, due + # to an rpm bug in versions prior to 4.20.0 + # + # https://github.com/rpm-software-management/rpm/issues/3057 + # https://github.com/rpm-software-management/rpm/commit/49f906998f3cf1f4152162ca61ac0869251c380f + # + # We cannot provide our own debuginfo package because it does not listen + # to our custom files list, failing the build due to unpackaged files. + # Manually generate the debug info package if using rpm 4.20.0. If not + # using rpm 4.20.0, avoid generating a -debuginfo package altogether, + # as it is not safe. + if grep -q CONFIG_MODULE_SIG=y include/config/auto.conf; then + rpm_ver_str=$(rpm --version 2>/dev/null) + # Split the version on spaces + IFS=' ' + set -- $rpm_ver_str + if [ "${1:-}" = RPM -a "${2:-}" = version ]; then + IFS=. + set -- $3 + rpm_ver=$(( 1000000 * $1 + 10000 * $2 + 100 * $3 + ${4:-0} )) + if [ "$rpm_ver" -ge 4200000 ]; then + with_debuginfo_manual='%{?_without_debuginfo:0}%{?!_without_debuginfo:1}' + fi + fi + else + with_debuginfo_rpm='%{?_without_debuginfo:0}%{?!_without_debuginfo:1}' + fi fi +echo "%define with_debuginfo_manual $with_debuginfo_manual" +echo "%define with_debuginfo_rpm $with_debuginfo_rpm" cat<f_path.mnt->mnt_sb; - load_root_writable = sb_is_writable(load_root); /* First loaded module/firmware defines the root for all others. */ spin_lock(&pinned_root_spinlock); @@ -169,8 +164,8 @@ static int loadpin_check(struct file *file, enum kernel_read_file_id id) spin_unlock(&pinned_root_spinlock); if (first_root_pin) { - report_writable(pinned_root, load_root_writable); - set_sysctl(load_root_writable); + loadpin_root_writable = sb_is_writable(pinned_root); + report_writable(pinned_root, loadpin_root_writable); report_load(origin, file, "pinned"); } diff --git a/tools/objtool/Makefile b/tools/objtool/Makefile index e0fa41933c18..61192f7778ff 100644 --- a/tools/objtool/Makefile +++ b/tools/objtool/Makefile @@ -29,6 +29,8 @@ srctree := $(patsubst %/,%,$(dir $(CURDIR))) srctree := $(patsubst %/,%,$(dir $(srctree))) endif +RM ?= rm -f + LIBSUBCMD_DIR = $(srctree)/tools/lib/subcmd/ ifneq ($(OUTPUT),) LIBSUBCMD_OUTPUT = $(abspath $(OUTPUT))/libsubcmd diff --git a/tools/power/x86/turbostat/turbostat.8 b/tools/power/x86/turbostat/turbostat.8 index 1551fcdbfd8a..344ede2f8546 100644 --- a/tools/power/x86/turbostat/turbostat.8 +++ b/tools/power/x86/turbostat/turbostat.8 @@ -111,10 +111,14 @@ The column name "all" can be used to enable all disabled-by-default built-in cou .PP \fB--no-perf\fP Disable all the uses of the perf API. .PP +\fB--force\fP Force turbostat to run on an unsupported platform (minimal defaults). +.PP \fB--interval seconds\fP overrides the default 5.0 second measurement interval. .PP \fB--num_iterations num\fP number of the measurement iterations. .PP +\fB--header_iterations num\fP print header every num iterations. +.PP \fB--out output_file\fP turbostat output is written to the specified output_file. The file is truncated if it already exists, and it is created if it does not exist. .PP @@ -159,15 +163,19 @@ The system configuration dump (if --quiet is not used) is followed by statistics .PP \fBSMI\fP The number of System Management Interrupts serviced CPU during the measurement interval. While this counter is actually per-CPU, SMI are triggered on all processors, so the number should be the same for all CPUs. .PP -\fBLLCkRPS\fP Last Level Cache Thousands of References Per Second. For CPUs with an L3 LLC, this is the number of references that CPU made to the L3 (and the number of misses that CPU made to it's L2). For CPUs with an L2 LLC, this is the number of references to the L2 (and the number of misses to the CPU's L1). The system summary row shows the sum for all CPUs. In both cases, the value displayed is the actual value divided by 1000 in the interest of usually fitting into 8 columns. +\fBLLCMRPS\fP Last Level Cache Millions of References Per Second. For CPUs with an L3 LLC, this is the number of references that CPU made to the L3 (and the number of misses that CPU made to it's L2). For CPUs with an L2 LLC, this is the number of references to the L2 (and the number of misses to the CPU's L1). The system summary row shows the sum for all CPUs. In both cases, the value displayed is the actual value divided by 1,000,000. If this value is large, then the LLC%hit column is significant. If this value is small, then the LLC%hit column is not significant. .PP -\fBLLC%hit\fP Last Level Cache Hit Rate %. Hit Rate Percent = 100.0 * (References - Misses)/References. The system summary row shows the weighted average for all CPUs (100.0 * (Sum_References - Sum_Misses)/Sum_References). +\fBLLC%hit\fP Last Level Cache Hit Rate %. Hit Rate Percent = 100.0 * Hits/References. The system summary row shows the weighted average for all CPUs (100.0 * Sum_Hits/Sum_References). +.PP +\fBL2MRPS\fP Level-2 Cache Millions of References Per Second. For CPUs with an L2 LLC, this is the same as LLC references. The system summary row shows the sum for all CPUs. In both cases, the value displayed is the actual value divided by 1,000,000. If this value is large, then the L2%hit column is significant. If this value is small, then the L2%hit column is not significant. +.PP +\fBL2%hit\fP Level-2 Cache Hit Rate %. Hit Rate Percent = 100.0 * Hits/References. The system summary row shows the weighted average for all CPUs (100.0 * (Sum_Hits)/Sum_References). .PP \fBC1, C2, C3...\fP The number times Linux requested the C1, C2, C3 idle state during the measurement interval. The system summary line shows the sum for all CPUs. These are C-state names as exported in /sys/devices/system/cpu/cpu*/cpuidle/state*/name. While their names are generic, their attributes are processor specific. They the system description section of output shows what MWAIT sub-states they are mapped to on each system. These counters are in the "cpuidle" group, which is disabled, by default. .PP -\fBC1+, C2+, C3+...\fP The idle governor idle state misprediction statistics. Inidcates the number times Linux requested the C1, C2, C3 idle state during the measurement interval, but should have requested a deeper idle state (if it exists and enabled). These statistics come from the /sys/devices/system/cpu/cpu*/cpuidle/state*/below file. These counters are in the "cpuidle" group, which is disabled, by default. +\fBC1+, C2+, C3+...\fP The idle governor idle state misprediction statistics. Indicates the number times Linux requested the C1, C2, C3 idle state during the measurement interval, but should have requested a deeper idle state (if it exists and enabled). These statistics come from the /sys/devices/system/cpu/cpu*/cpuidle/state*/below file. These counters are in the "cpuidle" group, which is disabled, by default. .PP -\fBC1-, C2-, C3-...\fP The idle governor idle state misprediction statistics. Inidcates the number times Linux requested the C1, C2, C3 idle state during the measurement interval, but should have requested a shallower idle state (if it exists and enabled). These statistics come from the /sys/devices/system/cpu/cpu*/cpuidle/state*/above file. These counters are in the "cpuidle" group, which is disabled, by default. +\fBC1-, C2-, C3-...\fP The idle governor idle state misprediction statistics. Indicates the number times Linux requested the C1, C2, C3 idle state during the measurement interval, but should have requested a shallower idle state (if it exists and enabled). These statistics come from the /sys/devices/system/cpu/cpu*/cpuidle/state*/above file. These counters are in the "cpuidle" group, which is disabled, by default. .PP \fBC1%, C2%, C3%\fP The residency percentage that Linux requested C1, C2, C3.... The system summary is the average of all CPUs in the system. Note that these are software, reflecting what was requested. The hardware counters reflect what was actually achieved. These counters are in the "pct_idle" group, which is enabled by default. .PP @@ -197,7 +205,7 @@ The system configuration dump (if --quiet is not used) is followed by statistics .PP \fBGFX%C0\fP Percentage of time that at least one GFX compute engine is busy. .PP -\fBCPUGFX%\fP Percentage of time that at least one CPU is busy at the same time as at least one Graphics compute enginer is busy. +\fBCPUGFX%\fP Percentage of time that at least one CPU is busy at the same time as at least one Graphics compute engine is busy. .PP \fBPkg%pc2, Pkg%pc3, Pkg%pc6, Pkg%pc7\fP percentage residency in hardware package idle states. These numbers are from hardware residency counters. .PP @@ -559,6 +567,8 @@ If the upstream version isn't new enough, the development tree can be found here If the development tree doesn't work, please contact the author via chat, or via email with the word "turbostat" on the Subject line. +An old turbostat binary may run on unknown hardware by using "--force", +but results are unsupported. .SH FILES .ta .nf diff --git a/tools/power/x86/turbostat/turbostat.c b/tools/power/x86/turbostat/turbostat.c index 5ad45c2ac5bd..1a2671c28209 100644 --- a/tools/power/x86/turbostat/turbostat.c +++ b/tools/power/x86/turbostat/turbostat.c @@ -3,7 +3,7 @@ * turbostat -- show CPU frequency and C-state residency * on modern Intel and AMD processors. * - * Copyright (c) 2025 Intel Corporation. + * Copyright (c) 2010 - 2026 Intel Corporation * Len Brown */ @@ -210,8 +210,10 @@ struct msr_counter bic[] = { { 0x0, "NMI", NULL, 0, 0, 0, NULL, 0 }, { 0x0, "CPU%c1e", NULL, 0, 0, 0, NULL, 0 }, { 0x0, "pct_idle", NULL, 0, 0, 0, NULL, 0 }, - { 0x0, "LLCkRPS", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "LLCMRPS", NULL, 0, 0, 0, NULL, 0 }, { 0x0, "LLC%hit", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "L2MRPS", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "L2%hit", NULL, 0, 0, 0, NULL, 0 }, }; /* n.b. bic_names must match the order in bic[], above */ @@ -281,8 +283,10 @@ enum bic_names { BIC_NMI, BIC_CPU_c1e, BIC_pct_idle, - BIC_LLC_RPS, + BIC_LLC_MRPS, BIC_LLC_HIT, + BIC_L2_MRPS, + BIC_L2_HIT, MAX_BIC }; @@ -294,12 +298,10 @@ void print_bic_set(char *s, cpu_set_t *set) printf("%s:", s); - for (i = 0; i <= MAX_BIC; ++i) { + for (i = 0; i < MAX_BIC; ++i) { - if (CPU_ISSET(i, set)) { - assert(i < MAX_BIC); + if (CPU_ISSET(i, set)) printf(" %s", bic[i].name); - } } putchar('\n'); } @@ -424,8 +426,10 @@ static void bic_groups_init(void) SET_BIC(BIC_pct_idle, &bic_group_idle); BIC_INIT(&bic_group_cache); - SET_BIC(BIC_LLC_RPS, &bic_group_cache); + SET_BIC(BIC_LLC_MRPS, &bic_group_cache); SET_BIC(BIC_LLC_HIT, &bic_group_cache); + SET_BIC(BIC_L2_MRPS, &bic_group_cache); + SET_BIC(BIC_L2_HIT, &bic_group_cache); BIC_INIT(&bic_group_other); SET_BIC(BIC_IRQ, &bic_group_other); @@ -482,6 +486,7 @@ FILE *outf; int *fd_percpu; int *fd_instr_count_percpu; int *fd_llc_percpu; +int *fd_l2_percpu; struct timeval interval_tv = { 5, 0 }; struct timespec interval_ts = { 5, 0 }; @@ -498,6 +503,7 @@ unsigned int list_header_only; unsigned int dump_only; unsigned int force_load; unsigned int cpuid_has_aperf_mperf; +unsigned int cpuid_has_hv; unsigned int has_aperf_access; unsigned int has_epb; unsigned int has_turbo; @@ -528,7 +534,7 @@ double rapl_dram_energy_units, rapl_energy_units, rapl_psys_energy_units; double rapl_joule_counter_range; unsigned int crystal_hz; unsigned long long tsc_hz; -int base_cpu; +int master_cpu; unsigned int has_hwp; /* IA32_PM_ENABLE, IA32_HWP_CAPABILITIES */ /* IA32_HWP_REQUEST, IA32_HWP_STATUS */ unsigned int has_hwp_notify; /* IA32_HWP_INTERRUPT */ @@ -620,7 +626,7 @@ double slm_bclk(void) unsigned int i; double freq; - if (get_msr(base_cpu, MSR_FSB_FREQ, &msr)) + if (get_msr(master_cpu, MSR_FSB_FREQ, &msr)) fprintf(outf, "SLM BCLK: unknown\n"); i = msr & 0xf; @@ -1248,6 +1254,84 @@ static const struct platform_data turbostat_pdata[] = { { 0, NULL }, }; +struct { + unsigned int uniform; + unsigned int pcore; + unsigned int ecore; + unsigned int lcore; +} perf_pmu_types; + +/* + * Events are enumerated in https://github.com/intel/perfmon + * and tools/perf/pmu-events/arch/x86/.../cache.json + */ +struct perf_l2_events { + unsigned long long refs; /* L2_REQUEST.ALL */ + unsigned long long hits; /* L2_REQUEST.HIT */ +}; + +struct perf_model_support { + unsigned int vfm; + struct perf_l2_events first; + struct perf_l2_events second; + struct perf_l2_events third; +} *perf_model_support; + +/* Perf Cache Events */ +#define PCE(ext_umask, umask) (((unsigned long long) ext_umask) << 40 | umask << 8 | 0x24) + +/* + * Enumerate up to three perf CPU PMU's in a system. + * The first, second, and third columns are populated without skipping, describing + * pcore, ecore, lcore PMUs, in order, if present. (The associated PMU "type" field is + * read from sysfs in all cases.) Eg. + * + * non-hybrid: + * GNR: pcore, {}, {} + * ADL-N: ecore, {}, {} + * hybrid: + * MTL: pcore, ecore, {}% + * ARL-H: pcore, ecore, lcore + * LNL: ecore, ecore%%, {} + * + * % MTL physical lcore share architecture and PMU with ecore, and are thus not enumerated separately. + * %% LNL physical lcore is enumerated by perf as ecore + */ +static struct perf_model_support turbostat_perf_model_support[] = { + { INTEL_SAPPHIRERAPIDS_X, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, {}, {} }, + { INTEL_EMERALDRAPIDS_X, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, {}, {} }, + { INTEL_GRANITERAPIDS_X, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, {}, {} }, + { INTEL_GRANITERAPIDS_D, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, {}, {} }, + { INTEL_DIAMONDRAPIDS_X, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, {}, {} }, + + { INTEL_ATOM_GRACEMONT, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {}, {} }, /* ADL-N */ + { INTEL_ATOM_CRESTMONT_X, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {}, {} }, /* SRF */ + { INTEL_ATOM_CRESTMONT, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {}, {} }, /* GRR */ + { INTEL_ATOM_DARKMONT_X, { PCE(0x01, 0xFF), PCE(0x01, 0xBF)}, {}, {} }, /* CWF */ + + { INTEL_ALDERLAKE, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} }, + { INTEL_ALDERLAKE, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} }, + { INTEL_ALDERLAKE_L, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} }, + { INTEL_RAPTORLAKE, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} }, + { INTEL_RAPTORLAKE_P, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} }, + { INTEL_RAPTORLAKE_S, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} }, + { INTEL_METEORLAKE_L, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} }, + { INTEL_METEORLAKE, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} }, + { INTEL_ARROWLAKE_U, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} }, + + { INTEL_LUNARLAKE_M, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, { PCE(0x00, 0x07), PCE(0x00, 0x02)}, {} }, + { INTEL_ARROWLAKE_H, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, { PCE(0x00, 0x07), PCE(0x00, 0x02)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)} }, + { INTEL_ARROWLAKE, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, { PCE(0x00, 0x07), PCE(0x00, 0x02)}, {} }, + + { INTEL_PANTHERLAKE_L, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, { PCE(0x01, 0xFF), PCE(0x01, 0xBF)}, {} }, + { INTEL_WILDCATLAKE_L, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, { PCE(0x01, 0xFF), PCE(0x01, 0xBF)}, {} }, + + { INTEL_NOVALAKE, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, { PCE(0x01, 0xFF), PCE(0x01, 0xBF)}, {} }, + { INTEL_NOVALAKE_L, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, { PCE(0x01, 0xFF), PCE(0x01, 0xBF)}, {} }, + + { 0, {}, {}, {} } +}; + static const struct platform_features *platform; void probe_platform_features(unsigned int family, unsigned int model) @@ -1291,6 +1375,21 @@ end: exit(1); } +void init_perf_model_support(unsigned int family, unsigned int model) +{ + int i; + + if (!genuine_intel) + return; + + for (i = 0; turbostat_perf_model_support[i].vfm; i++) { + if (VFM_FAMILY(turbostat_perf_model_support[i].vfm) == family && VFM_MODEL(turbostat_perf_model_support[i].vfm) == model) { + perf_model_support = &turbostat_perf_model_support[i]; + return; + } + } +} + /* Model specific support End */ #define TJMAX_DEFAULT 100 @@ -1307,6 +1406,7 @@ char *progname; #define CPU_SUBSET_MAXCPUS 8192 /* need to use before probe... */ cpu_set_t *cpu_present_set, *cpu_possible_set, *cpu_effective_set, *cpu_allowed_set, *cpu_affinity_set, *cpu_subset; +cpu_set_t *perf_pcore_set, *perf_ecore_set, *perf_lcore_set; size_t cpu_present_setsize, cpu_possible_setsize, cpu_effective_setsize, cpu_allowed_setsize, cpu_affinity_setsize, cpu_subset_size; #define MAX_ADDED_THREAD_COUNTERS 24 #define MAX_ADDED_CORE_COUNTERS 8 @@ -2007,6 +2107,10 @@ struct llc_stats { unsigned long long references; unsigned long long misses; }; +struct l2_stats { + unsigned long long references; + unsigned long long hits; +}; struct thread_data { struct timeval tv_begin; struct timeval tv_end; @@ -2020,6 +2124,7 @@ struct thread_data { unsigned long long nmi_count; unsigned int smi_count; struct llc_stats llc; + struct l2_stats l2; unsigned int cpu_id; unsigned int apic_id; unsigned int x2apic_id; @@ -2028,25 +2133,24 @@ struct thread_data { unsigned long long counter[MAX_ADDED_THREAD_COUNTERS]; unsigned long long perf_counter[MAX_ADDED_THREAD_COUNTERS]; unsigned long long pmt_counter[PMT_MAX_ADDED_THREAD_COUNTERS]; -} *thread_even, *thread_odd; +}; struct core_data { - int base_cpu; + int first_cpu; unsigned long long c3; unsigned long long c6; unsigned long long c7; unsigned long long mc6_us; /* duplicate as per-core for now, even though per module */ unsigned int core_temp_c; struct rapl_counter core_energy; /* MSR_CORE_ENERGY_STAT */ - unsigned int core_id; unsigned long long core_throt_cnt; unsigned long long counter[MAX_ADDED_CORE_COUNTERS]; unsigned long long perf_counter[MAX_ADDED_CORE_COUNTERS]; unsigned long long pmt_counter[PMT_MAX_ADDED_CORE_COUNTERS]; -} *core_even, *core_odd; +}; struct pkg_data { - int base_cpu; + int first_cpu; unsigned long long pc2; unsigned long long pc3; unsigned long long pc6; @@ -2066,7 +2170,6 @@ struct pkg_data { long long sam_mc6_ms; unsigned int sam_mhz; unsigned int sam_act_mhz; - unsigned int package_id; struct rapl_counter energy_pkg; /* MSR_PKG_ENERGY_STATUS */ struct rapl_counter energy_dram; /* MSR_DRAM_ENERGY_STATUS */ struct rapl_counter energy_cores; /* MSR_PP0_ENERGY_STATUS */ @@ -2079,24 +2182,10 @@ struct pkg_data { unsigned long long counter[MAX_ADDED_PACKAGE_COUNTERS]; unsigned long long perf_counter[MAX_ADDED_PACKAGE_COUNTERS]; unsigned long long pmt_counter[PMT_MAX_ADDED_PACKAGE_COUNTERS]; -} *package_even, *package_odd; +}; -#define ODD_COUNTERS thread_odd, core_odd, package_odd -#define EVEN_COUNTERS thread_even, core_even, package_even - -#define GET_THREAD(thread_base, thread_no, core_no, node_no, pkg_no) \ - ((thread_base) + \ - ((pkg_no) * \ - topo.nodes_per_pkg * topo.cores_per_node * topo.threads_per_core) + \ - ((node_no) * topo.cores_per_node * topo.threads_per_core) + \ - ((core_no) * topo.threads_per_core) + \ - (thread_no)) - -#define GET_CORE(core_base, core_no, node_no, pkg_no) \ - ((core_base) + \ - ((pkg_no) * topo.nodes_per_pkg * topo.cores_per_node) + \ - ((node_no) * topo.cores_per_node) + \ - (core_no)) +#define ODD_COUNTERS odd.threads, odd.cores, odd.packages +#define EVEN_COUNTERS even.threads, even.cores, even.packages /* * The accumulated sum of MSR is defined as a monotonic @@ -2135,7 +2224,7 @@ off_t idx_to_offset(int idx) switch (idx) { case IDX_PKG_ENERGY: - if (valid_rapl_msrs & RAPL_AMD_F17H) + if (platform->plat_rapl_msrs & RAPL_AMD_F17H) offset = MSR_PKG_ENERGY_STAT; else offset = MSR_PKG_ENERGY_STATUS; @@ -2279,25 +2368,28 @@ static void free_sys_msr_counters(void) sys.added_package_counters -= free_msr_counters_(&sys.pp); } -struct system_summary { - struct thread_data threads; - struct core_data cores; - struct pkg_data packages; -} average; +struct counters { + struct thread_data *threads; + struct core_data *cores; + struct pkg_data *packages; +} average, even, odd; struct platform_counters { struct rapl_counter energy_psys; /* MSR_PLATFORM_ENERGY_STATUS */ } platform_counters_odd, platform_counters_even; +#define MAX_HT_ID 3 /* support SMT-4 */ + struct cpu_topology { - int physical_package_id; + int cpu_id; + int core_id; /* unique within a package */ + int package_id; int die_id; int l3_id; - int logical_cpu_id; int physical_node_id; int logical_node_id; /* 0-based count within the package */ - int physical_core_id; - int thread_id; + int ht_id; /* unique within a core */ + int ht_sibling_cpu_id[MAX_HT_ID + 1]; int type; cpu_set_t *put_ids; /* Processing Unit/Thread IDs */ } *cpus; @@ -2306,12 +2398,12 @@ struct topo_params { int num_packages; int num_die; int num_cpus; - int num_cores; + int num_cores; /* system wide */ int allowed_packages; int allowed_cpus; int allowed_cores; int max_cpu_num; - int max_core_id; + int max_core_id; /* within a package */ int max_package_id; int max_die_id; int max_l3_id; @@ -2343,6 +2435,7 @@ int cpu_is_not_allowed(int cpu) return !CPU_ISSET_S(cpu, cpu_allowed_setsize, cpu_allowed_set); } +#define GLOBAL_CORE_ID(core_id, pkg_id) (core_id + pkg_id * (topo.max_core_id + 1)) /* * run func(thread, core, package) in topology order * skip non-present cpus @@ -2353,27 +2446,38 @@ int cpu_is_not_allowed(int cpu) int for_all_cpus(int (func) (struct thread_data *, struct core_data *, struct pkg_data *), struct thread_data *thread_base, struct core_data *core_base, struct pkg_data *pkg_base) { - int retval, pkg_no, core_no, thread_no, node_no; + int cpu, retval; retval = 0; - for (pkg_no = 0; pkg_no < topo.num_packages; ++pkg_no) { - for (node_no = 0; node_no < topo.nodes_per_pkg; node_no++) { - for (core_no = 0; core_no < topo.cores_per_node; ++core_no) { - for (thread_no = 0; thread_no < topo.threads_per_core; ++thread_no) { - struct thread_data *t; - struct core_data *c; + for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu) { + struct thread_data *t; + struct core_data *c; + struct pkg_data *p; - t = GET_THREAD(thread_base, thread_no, core_no, node_no, pkg_no); + int pkg_id = cpus[cpu].package_id; - if (cpu_is_not_allowed(t->cpu_id)) - continue; + if (cpu_is_not_allowed(cpu)) + continue; - c = GET_CORE(core_base, core_no, node_no, pkg_no); + if (cpus[cpu].ht_id > 0) /* skip HT sibling */ + continue; - retval |= func(t, c, &pkg_base[pkg_no]); - } - } + t = &thread_base[cpu]; + c = &core_base[GLOBAL_CORE_ID(cpus[cpu].core_id, pkg_id)]; + p = &pkg_base[pkg_id]; + + retval |= func(t, c, p); + + /* Handle HT sibling now */ + int i; + + for (i = MAX_HT_ID; i > 0; --i) { /* ht_id 0 is self */ + if (cpus[cpu].ht_sibling_cpu_id[i] <= 0) + continue; + t = &thread_base[cpus[cpu].ht_sibling_cpu_id[i]]; + + retval |= func(t, c, p); } } return retval; @@ -2381,12 +2485,12 @@ int for_all_cpus(int (func) (struct thread_data *, struct core_data *, struct pk int is_cpu_first_thread_in_core(struct thread_data *t, struct core_data *c) { - return ((int)t->cpu_id == c->base_cpu || c->base_cpu < 0); + return ((int)t->cpu_id == c->first_cpu || c->first_cpu < 0); } int is_cpu_first_core_in_package(struct thread_data *t, struct pkg_data *p) { - return ((int)t->cpu_id == p->base_cpu || p->base_cpu < 0); + return ((int)t->cpu_id == p->first_cpu || p->first_cpu < 0); } int is_cpu_first_thread_in_package(struct thread_data *t, struct core_data *c, struct pkg_data *p) @@ -2439,8 +2543,10 @@ static void bic_disable_msr_access(void) static void bic_disable_perf_access(void) { CLR_BIC(BIC_IPC, &bic_enabled); - CLR_BIC(BIC_LLC_RPS, &bic_enabled); + CLR_BIC(BIC_LLC_MRPS, &bic_enabled); CLR_BIC(BIC_LLC_HIT, &bic_enabled); + CLR_BIC(BIC_L2_MRPS, &bic_enabled); + CLR_BIC(BIC_L2_HIT, &bic_enabled); } static long perf_event_open(struct perf_event_attr *hw_event, pid_t pid, int cpu, int group_fd, unsigned long flags) @@ -2552,10 +2658,10 @@ unsigned int cpu_to_domain(const struct perf_counter_info *pc, int cpu) return cpu; case SCOPE_CORE: - return cpus[cpu].physical_core_id; + return cpus[cpu].core_id; case SCOPE_PACKAGE: - return cpus[cpu].physical_package_id; + return cpus[cpu].package_id; } __builtin_unreachable(); @@ -2629,8 +2735,7 @@ void help(void) " sets the Thermal Control Circuit temperature in\n" " degrees Celsius\n" " -h, --help\n" - " print this help message\n" - " -v, --version\n\t\tprint version information\n\nFor more help, run \"man turbostat\"\n"); + " print this help message\n -v, --version\n\t\tprint version information\n\nFor more help, run \"man turbostat\"\n"); } /* @@ -2813,12 +2918,18 @@ void print_header(char *delim) if (DO_BIC(BIC_SMI)) outp += sprintf(outp, "%sSMI", (printed++ ? delim : "")); - if (DO_BIC(BIC_LLC_RPS)) - outp += sprintf(outp, "%sLLCkRPS", (printed++ ? delim : "")); + if (DO_BIC(BIC_LLC_MRPS)) + outp += sprintf(outp, "%sLLCMRPS", (printed++ ? delim : "")); if (DO_BIC(BIC_LLC_HIT)) outp += sprintf(outp, "%sLLC%%hit", (printed++ ? delim : "")); + if (DO_BIC(BIC_L2_MRPS)) + outp += sprintf(outp, "%sL2MRPS", (printed++ ? delim : "")); + + if (DO_BIC(BIC_L2_HIT)) + outp += sprintf(outp, "%sL2%%hit", (printed++ ? delim : "")); + for (mp = sys.tp; mp; mp = mp->next) outp += print_name(mp->width, &printed, delim, mp->name, mp->type, mp->format); @@ -3001,29 +3112,37 @@ void print_header(char *delim) } /* - * pct() + * pct(numerator, denominator) * - * If absolute value is < 1.1, return percentage - * otherwise, return nan + * Return sanity checked percentage (100.0 * numerator/denominotor) * - * return value is appropriate for printing percentages with %f - * while flagging some obvious erroneous values. + * n < 0: nan + * d <= 0: nan + * n/d > 1.1: nan */ -double pct(double d) +double pct(double numerator, double denominator) { + double retval; - double abs = fabs(d); + if (numerator < 0) + return nan(""); - if (abs < 1.10) - return (100.0 * d); - return nan(""); + if (denominator <= 0) + return nan(""); + + retval = 100.0 * numerator / denominator; + + if (retval > 110.0) + return nan(""); + + return retval; } int dump_counters(PER_THREAD_PARAMS) { int i; struct msr_counter *mp; - struct platform_counters *pplat_cnt = p == package_odd ? &platform_counters_odd : &platform_counters_even; + struct platform_counters *pplat_cnt = p == odd.packages ? &platform_counters_odd : &platform_counters_even; outp += sprintf(outp, "t %p, c %p, p %p\n", t, c, p); @@ -3046,7 +3165,11 @@ int dump_counters(PER_THREAD_PARAMS) outp += sprintf(outp, "LLC refs: %lld", t->llc.references); outp += sprintf(outp, "LLC miss: %lld", t->llc.misses); - outp += sprintf(outp, "LLC Hit%%: %.2f", pct((t->llc.references - t->llc.misses) / t->llc.references)); + outp += sprintf(outp, "LLC Hit%%: %.2f", pct((t->llc.references - t->llc.misses), t->llc.references)); + + outp += sprintf(outp, "L2 refs: %lld", t->l2.references); + outp += sprintf(outp, "L2 hits: %lld", t->l2.hits); + outp += sprintf(outp, "L2 Hit%%: %.2f", pct(t->l2.hits, t->l2.references)); for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { outp += sprintf(outp, "tADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, t->counter[i], mp->sp->path); @@ -3054,7 +3177,7 @@ int dump_counters(PER_THREAD_PARAMS) } if (c && is_cpu_first_thread_in_core(t, c)) { - outp += sprintf(outp, "core: %d\n", c->core_id); + outp += sprintf(outp, "core: %d\n", cpus[t->cpu_id].core_id); outp += sprintf(outp, "c3: %016llX\n", c->c3); outp += sprintf(outp, "c6: %016llX\n", c->c6); outp += sprintf(outp, "c7: %016llX\n", c->c7); @@ -3074,8 +3197,6 @@ int dump_counters(PER_THREAD_PARAMS) } if (p && is_cpu_first_core_in_package(t, p)) { - outp += sprintf(outp, "package: %d\n", p->package_id); - outp += sprintf(outp, "Weighted cores: %016llX\n", p->pkg_wtd_core_c0); outp += sprintf(outp, "Any cores: %016llX\n", p->pkg_any_core_c0); outp += sprintf(outp, "Any GFX: %016llX\n", p->pkg_any_gfxe_c0); @@ -3141,7 +3262,7 @@ void get_perf_llc_stats(int cpu, struct llc_stats *llc) actual_read_size = read(fd_llc_percpu[cpu], &r, expected_read_size); if (actual_read_size == -1) - err(-1, "%s(cpu%d,) %d,,%ld\n", __func__, cpu, fd_llc_percpu[cpu], expected_read_size); + err(-1, "%s(cpu%d,) %d,,%ld", __func__, cpu, fd_llc_percpu[cpu], expected_read_size); llc->references = r.llc.references; llc->misses = r.llc.misses; @@ -3149,6 +3270,26 @@ void get_perf_llc_stats(int cpu, struct llc_stats *llc) warn("%s: failed to read perf_data (req %zu act %zu)", __func__, expected_read_size, actual_read_size); } +void get_perf_l2_stats(int cpu, struct l2_stats *l2) +{ + struct read_format { + unsigned long long num_read; + struct l2_stats l2; + } r; + const ssize_t expected_read_size = sizeof(r); + ssize_t actual_read_size; + + actual_read_size = read(fd_l2_percpu[cpu], &r, expected_read_size); + + if (actual_read_size == -1) + err(-1, "%s(cpu%d,) %d,,%ld", __func__, cpu, fd_l2_percpu[cpu], expected_read_size); + + l2->references = r.l2.references; + l2->hits = r.l2.hits; + if (actual_read_size != expected_read_size) + warn("%s: cpu%d: failed to read(%d) perf_data (req %zu act %zu)", __func__, cpu, fd_l2_percpu[cpu], expected_read_size, actual_read_size); +} + /* * column formatting convention & formats */ @@ -3167,7 +3308,7 @@ int format_counters(PER_THREAD_PARAMS) char *delim = "\t"; int printed = 0; - if (t == &average.threads) { + if (t == average.threads) { pplat_cnt = count & 1 ? &platform_counters_odd : &platform_counters_even; ++count; } @@ -3181,7 +3322,7 @@ int format_counters(PER_THREAD_PARAMS) return 0; /*if not summary line and --cpu is used */ - if ((t != &average.threads) && (cpu_subset && !CPU_ISSET_S(t->cpu_id, cpu_subset_size, cpu_subset))) + if ((t != average.threads) && (cpu_subset && !CPU_ISSET_S(t->cpu_id, cpu_subset_size, cpu_subset))) return 0; if (DO_BIC(BIC_USEC)) { @@ -3201,7 +3342,7 @@ int format_counters(PER_THREAD_PARAMS) tsc = t->tsc * tsc_tweak; /* topo columns, print blanks on 1st (average) line */ - if (t == &average.threads) { + if (t == average.threads) { if (DO_BIC(BIC_Package)) outp += sprintf(outp, "%s-", (printed++ ? delim : "")); if (DO_BIC(BIC_Die)) @@ -3221,7 +3362,7 @@ int format_counters(PER_THREAD_PARAMS) } else { if (DO_BIC(BIC_Package)) { if (p) - outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->package_id); + outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), cpus[t->cpu_id].package_id); else outp += sprintf(outp, "%s-", (printed++ ? delim : "")); } @@ -3245,7 +3386,7 @@ int format_counters(PER_THREAD_PARAMS) } if (DO_BIC(BIC_Core)) { if (c) - outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), c->core_id); + outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), cpus[t->cpu_id].core_id); else outp += sprintf(outp, "%s-", (printed++ ? delim : "")); } @@ -3261,7 +3402,7 @@ int format_counters(PER_THREAD_PARAMS) outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), 1.0 / units * t->aperf / interval_float); if (DO_BIC(BIC_Busy)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(t->mperf / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(t->mperf, tsc)); if (DO_BIC(BIC_Bzy_MHz)) { if (has_base_hz) @@ -3297,13 +3438,18 @@ int format_counters(PER_THREAD_PARAMS) outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), t->smi_count); /* LLC Stats */ - if (DO_BIC(BIC_LLC_RPS) || DO_BIC(BIC_LLC_HIT)) { - if (DO_BIC(BIC_LLC_RPS)) - outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), t->llc.references / interval_float / 1000); + if (DO_BIC(BIC_LLC_MRPS)) + outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), t->llc.references / interval_float / 1000000); - if (DO_BIC(BIC_LLC_HIT)) - outp += sprintf(outp, fmt8, (printed++ ? delim : ""), pct((t->llc.references - t->llc.misses) / t->llc.references)); - } + if (DO_BIC(BIC_LLC_HIT)) + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), pct((t->llc.references - t->llc.misses), t->llc.references)); + + /* L2 Stats */ + if (DO_BIC(BIC_L2_MRPS)) + outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), t->l2.references / interval_float / 1000000); + + if (DO_BIC(BIC_L2_HIT)) + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), pct(t->l2.hits, t->l2.references)); /* Added Thread Counters */ for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { @@ -3315,7 +3461,7 @@ int format_counters(PER_THREAD_PARAMS) if (mp->type == COUNTER_USEC) outp += print_float_value(&printed, delim, t->counter[i] / interval_float / 10000); else - outp += print_float_value(&printed, delim, pct(t->counter[i] / tsc)); + outp += print_float_value(&printed, delim, pct(t->counter[i], tsc)); } } @@ -3329,7 +3475,7 @@ int format_counters(PER_THREAD_PARAMS) if (pp->type == COUNTER_USEC) outp += print_float_value(&printed, delim, t->perf_counter[i] / interval_float / 10000); else - outp += print_float_value(&printed, delim, pct(t->perf_counter[i] / tsc)); + outp += print_float_value(&printed, delim, pct(t->perf_counter[i], tsc)); } } @@ -3343,34 +3489,34 @@ int format_counters(PER_THREAD_PARAMS) break; case PMT_TYPE_XTAL_TIME: - value_converted = pct(value_raw / crystal_hz / interval_float); + value_converted = pct(value_raw / crystal_hz, interval_float); outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), value_converted); break; case PMT_TYPE_TCORE_CLOCK: - value_converted = pct(value_raw / tcore_clock_freq_hz / interval_float); + value_converted = pct(value_raw / tcore_clock_freq_hz, interval_float); outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), value_converted); } } /* C1 */ if (DO_BIC(BIC_CPU_c1)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(t->c1 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(t->c1, tsc)); /* print per-core data only for 1st thread in core */ if (!is_cpu_first_thread_in_core(t, c)) goto done; if (DO_BIC(BIC_CPU_c3)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c3 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c3, tsc)); if (DO_BIC(BIC_CPU_c6)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c6 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c6, tsc)); if (DO_BIC(BIC_CPU_c7)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c7 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c7, tsc)); /* Mod%c6 */ if (DO_BIC(BIC_Mod_c6)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->mc6_us / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->mc6_us, tsc)); if (DO_BIC(BIC_CoreTmp)) outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), c->core_temp_c); @@ -3386,7 +3532,7 @@ int format_counters(PER_THREAD_PARAMS) else if (mp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE) outp += print_decimal_value(mp->width, &printed, delim, c->counter[i]); else if (mp->format == FORMAT_PERCENT) - outp += print_float_value(&printed, delim, pct(c->counter[i] / tsc)); + outp += print_float_value(&printed, delim, pct(c->counter[i], tsc)); } /* Added perf Core counters */ @@ -3396,7 +3542,7 @@ int format_counters(PER_THREAD_PARAMS) else if (pp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE) outp += print_decimal_value(pp->width, &printed, delim, c->perf_counter[i]); else if (pp->format == FORMAT_PERCENT) - outp += print_float_value(&printed, delim, pct(c->perf_counter[i] / tsc)); + outp += print_float_value(&printed, delim, pct(c->perf_counter[i], tsc)); } /* Added PMT Core counters */ @@ -3409,12 +3555,12 @@ int format_counters(PER_THREAD_PARAMS) break; case PMT_TYPE_XTAL_TIME: - value_converted = pct(value_raw / crystal_hz / interval_float); + value_converted = pct(value_raw / crystal_hz, interval_float); outp += print_float_value(&printed, delim, value_converted); break; case PMT_TYPE_TCORE_CLOCK: - value_converted = pct(value_raw / tcore_clock_freq_hz / interval_float); + value_converted = pct(value_raw / tcore_clock_freq_hz, interval_float); outp += print_float_value(&printed, delim, value_converted); } } @@ -3470,39 +3616,39 @@ int format_counters(PER_THREAD_PARAMS) if (DO_BIC(BIC_Totl_c0)) outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100 * p->pkg_wtd_core_c0 / tsc); /* can exceed 100% */ if (DO_BIC(BIC_Any_c0)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_any_core_c0 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_any_core_c0, tsc)); if (DO_BIC(BIC_GFX_c0)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_any_gfxe_c0 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_any_gfxe_c0, tsc)); if (DO_BIC(BIC_CPUGFX)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_both_core_gfxe_c0 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_both_core_gfxe_c0, tsc)); if (DO_BIC(BIC_Pkgpc2)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc2 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc2, tsc)); if (DO_BIC(BIC_Pkgpc3)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc3 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc3, tsc)); if (DO_BIC(BIC_Pkgpc6)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc6 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc6, tsc)); if (DO_BIC(BIC_Pkgpc7)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc7 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc7, tsc)); if (DO_BIC(BIC_Pkgpc8)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc8 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc8, tsc)); if (DO_BIC(BIC_Pkgpc9)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc9 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc9, tsc)); if (DO_BIC(BIC_Pkgpc10)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc10 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc10, tsc)); if (DO_BIC(BIC_Diec6)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->die_c6 / crystal_hz / interval_float)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->die_c6 / crystal_hz, interval_float)); if (DO_BIC(BIC_CPU_LPI)) { if (p->cpu_lpi >= 0) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->cpu_lpi / 1000000.0 / interval_float)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->cpu_lpi / 1000000.0, interval_float)); else outp += sprintf(outp, "%s(neg)", (printed++ ? delim : "")); } if (DO_BIC(BIC_SYS_LPI)) { if (p->sys_lpi >= 0) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->sys_lpi / 1000000.0 / interval_float)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->sys_lpi / 1000000.0, interval_float)); else outp += sprintf(outp, "%s(neg)", (printed++ ? delim : "")); } @@ -3524,11 +3670,9 @@ int format_counters(PER_THREAD_PARAMS) if (DO_BIC(BIC_RAM_J)) outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_dram, RAPL_UNIT_JOULES, interval_float)); if (DO_BIC(BIC_PKG__)) - outp += - sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->rapl_pkg_perf_status, RAPL_UNIT_WATTS, interval_float)); + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->rapl_pkg_perf_status, RAPL_UNIT_WATTS, interval_float)); if (DO_BIC(BIC_RAM__)) - outp += - sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->rapl_dram_perf_status, RAPL_UNIT_WATTS, interval_float)); + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->rapl_dram_perf_status, RAPL_UNIT_WATTS, interval_float)); /* UncMHz */ if (DO_BIC(BIC_UNCORE_MHZ)) outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->uncore_mhz); @@ -3542,7 +3686,7 @@ int format_counters(PER_THREAD_PARAMS) else if (mp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE) outp += print_decimal_value(mp->width, &printed, delim, p->counter[i]); else if (mp->format == FORMAT_PERCENT) - outp += print_float_value(&printed, delim, pct(p->counter[i] / tsc)); + outp += print_float_value(&printed, delim, pct(p->counter[i], tsc)); } /* Added perf Package Counters */ @@ -3554,7 +3698,7 @@ int format_counters(PER_THREAD_PARAMS) else if (pp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE) outp += print_decimal_value(pp->width, &printed, delim, p->perf_counter[i]); else if (pp->format == FORMAT_PERCENT) - outp += print_float_value(&printed, delim, pct(p->perf_counter[i] / tsc)); + outp += print_float_value(&printed, delim, pct(p->perf_counter[i], tsc)); } /* Added PMT Package Counters */ @@ -3567,22 +3711,20 @@ int format_counters(PER_THREAD_PARAMS) break; case PMT_TYPE_XTAL_TIME: - value_converted = pct(value_raw / crystal_hz / interval_float); + value_converted = pct(value_raw / crystal_hz, interval_float); outp += print_float_value(&printed, delim, value_converted); break; case PMT_TYPE_TCORE_CLOCK: - value_converted = pct(value_raw / tcore_clock_freq_hz / interval_float); + value_converted = pct(value_raw / tcore_clock_freq_hz, interval_float); outp += print_float_value(&printed, delim, value_converted); } } - if (DO_BIC(BIC_SysWatt) && (t == &average.threads)) - outp += sprintf(outp, fmt8, (printed++ ? delim : ""), - rapl_counter_get_value(&pplat_cnt->energy_psys, RAPL_UNIT_WATTS, interval_float)); - if (DO_BIC(BIC_Sys_J) && (t == &average.threads)) - outp += sprintf(outp, fmt8, (printed++ ? delim : ""), - rapl_counter_get_value(&pplat_cnt->energy_psys, RAPL_UNIT_JOULES, interval_float)); + if (DO_BIC(BIC_SysWatt) && (t == average.threads)) + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&pplat_cnt->energy_psys, RAPL_UNIT_WATTS, interval_float)); + if (DO_BIC(BIC_Sys_J) && (t == average.threads)) + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&pplat_cnt->energy_psys, RAPL_UNIT_JOULES, interval_float)); done: if (*(outp - 1) != '\n') @@ -3620,7 +3762,7 @@ void format_all_counters(PER_THREAD_PARAMS) if ((!count || (header_iterations && !(count % header_iterations))) || !summary_only) print_header("\t"); - format_counters(&average.threads, &average.cores, &average.packages); + format_counters(average.threads, average.cores, average.packages); count++; @@ -3795,7 +3937,7 @@ int delta_thread(struct thread_data *new, struct thread_data *old, struct core_d /* check for TSC < 1 Mcycles over interval */ if (old->tsc < (1000 * 1000)) errx(-3, "Insanely slow TSC rate, TSC stops in idle?\n" - "You can disable all c-states by booting with \"idle=poll\"\n" "or just the deep ones with \"processor.max_cstate=1\""); + "You can disable all c-states by booting with \"idle=poll\"\nor just the deep ones with \"processor.max_cstate=1\""); old->c1 = new->c1 - old->c1; @@ -3846,12 +3988,18 @@ int delta_thread(struct thread_data *new, struct thread_data *old, struct core_d if (DO_BIC(BIC_SMI)) old->smi_count = new->smi_count - old->smi_count; - if (DO_BIC(BIC_LLC_RPS)) + if (DO_BIC(BIC_LLC_MRPS) || DO_BIC(BIC_LLC_HIT)) old->llc.references = new->llc.references - old->llc.references; if (DO_BIC(BIC_LLC_HIT)) old->llc.misses = new->llc.misses - old->llc.misses; + if (DO_BIC(BIC_L2_MRPS) || DO_BIC(BIC_L2_HIT)) + old->l2.references = new->l2.references - old->l2.references; + + if (DO_BIC(BIC_L2_HIT)) + old->l2.hits = new->l2.hits - old->l2.hits; + for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { if (mp->format == FORMAT_RAW || mp->format == FORMAT_AVERAGE) old->counter[i] = new->counter[i]; @@ -3932,6 +4080,9 @@ void clear_counters(PER_THREAD_PARAMS) t->llc.references = 0; t->llc.misses = 0; + t->l2.references = 0; + t->l2.hits = 0; + c->c3 = 0; c->c6 = 0; c->c7 = 0; @@ -3940,9 +4091,6 @@ void clear_counters(PER_THREAD_PARAMS) rapl_counter_clear(&c->core_energy); c->core_throt_cnt = 0; - t->llc.references = 0; - t->llc.misses = 0; - p->pkg_wtd_core_c0 = 0; p->pkg_any_core_c0 = 0; p->pkg_any_gfxe_c0 = 0; @@ -4018,75 +4166,78 @@ int sum_counters(PER_THREAD_PARAMS) /* copy un-changing apic_id's */ if (DO_BIC(BIC_APIC)) - average.threads.apic_id = t->apic_id; + average.threads->apic_id = t->apic_id; if (DO_BIC(BIC_X2APIC)) - average.threads.x2apic_id = t->x2apic_id; + average.threads->x2apic_id = t->x2apic_id; /* remember first tv_begin */ - if (average.threads.tv_begin.tv_sec == 0) - average.threads.tv_begin = procsysfs_tv_begin; + if (average.threads->tv_begin.tv_sec == 0) + average.threads->tv_begin = procsysfs_tv_begin; /* remember last tv_end */ - average.threads.tv_end = t->tv_end; + average.threads->tv_end = t->tv_end; - average.threads.tsc += t->tsc; - average.threads.aperf += t->aperf; - average.threads.mperf += t->mperf; - average.threads.c1 += t->c1; + average.threads->tsc += t->tsc; + average.threads->aperf += t->aperf; + average.threads->mperf += t->mperf; + average.threads->c1 += t->c1; - average.threads.instr_count += t->instr_count; + average.threads->instr_count += t->instr_count; - average.threads.irq_count += t->irq_count; - average.threads.nmi_count += t->nmi_count; - average.threads.smi_count += t->smi_count; + average.threads->irq_count += t->irq_count; + average.threads->nmi_count += t->nmi_count; + average.threads->smi_count += t->smi_count; - average.threads.llc.references += t->llc.references; - average.threads.llc.misses += t->llc.misses; + average.threads->llc.references += t->llc.references; + average.threads->llc.misses += t->llc.misses; + + average.threads->l2.references += t->l2.references; + average.threads->l2.hits += t->l2.hits; for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { if (mp->format == FORMAT_RAW) continue; - average.threads.counter[i] += t->counter[i]; + average.threads->counter[i] += t->counter[i]; } for (i = 0, pp = sys.perf_tp; pp; i++, pp = pp->next) { if (pp->format == FORMAT_RAW) continue; - average.threads.perf_counter[i] += t->perf_counter[i]; + average.threads->perf_counter[i] += t->perf_counter[i]; } for (i = 0, ppmt = sys.pmt_tp; ppmt; i++, ppmt = ppmt->next) { - average.threads.pmt_counter[i] += t->pmt_counter[i]; + average.threads->pmt_counter[i] += t->pmt_counter[i]; } /* sum per-core values only for 1st thread in core */ if (!is_cpu_first_thread_in_core(t, c)) return 0; - average.cores.c3 += c->c3; - average.cores.c6 += c->c6; - average.cores.c7 += c->c7; - average.cores.mc6_us += c->mc6_us; + average.cores->c3 += c->c3; + average.cores->c6 += c->c6; + average.cores->c7 += c->c7; + average.cores->mc6_us += c->mc6_us; - average.cores.core_temp_c = MAX(average.cores.core_temp_c, c->core_temp_c); - average.cores.core_throt_cnt = MAX(average.cores.core_throt_cnt, c->core_throt_cnt); + average.cores->core_temp_c = MAX(average.cores->core_temp_c, c->core_temp_c); + average.cores->core_throt_cnt = MAX(average.cores->core_throt_cnt, c->core_throt_cnt); - rapl_counter_accumulate(&average.cores.core_energy, &c->core_energy); + rapl_counter_accumulate(&average.cores->core_energy, &c->core_energy); for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) { if (mp->format == FORMAT_RAW) continue; - average.cores.counter[i] += c->counter[i]; + average.cores->counter[i] += c->counter[i]; } for (i = 0, pp = sys.perf_cp; pp; i++, pp = pp->next) { if (pp->format == FORMAT_RAW) continue; - average.cores.perf_counter[i] += c->perf_counter[i]; + average.cores->perf_counter[i] += c->perf_counter[i]; } for (i = 0, ppmt = sys.pmt_cp; ppmt; i++, ppmt = ppmt->next) { - average.cores.pmt_counter[i] += c->pmt_counter[i]; + average.cores->pmt_counter[i] += c->pmt_counter[i]; } /* sum per-pkg values only for 1st core in pkg */ @@ -4094,63 +4245,63 @@ int sum_counters(PER_THREAD_PARAMS) return 0; if (DO_BIC(BIC_Totl_c0)) - average.packages.pkg_wtd_core_c0 += p->pkg_wtd_core_c0; + average.packages->pkg_wtd_core_c0 += p->pkg_wtd_core_c0; if (DO_BIC(BIC_Any_c0)) - average.packages.pkg_any_core_c0 += p->pkg_any_core_c0; + average.packages->pkg_any_core_c0 += p->pkg_any_core_c0; if (DO_BIC(BIC_GFX_c0)) - average.packages.pkg_any_gfxe_c0 += p->pkg_any_gfxe_c0; + average.packages->pkg_any_gfxe_c0 += p->pkg_any_gfxe_c0; if (DO_BIC(BIC_CPUGFX)) - average.packages.pkg_both_core_gfxe_c0 += p->pkg_both_core_gfxe_c0; + average.packages->pkg_both_core_gfxe_c0 += p->pkg_both_core_gfxe_c0; - average.packages.pc2 += p->pc2; + average.packages->pc2 += p->pc2; if (DO_BIC(BIC_Pkgpc3)) - average.packages.pc3 += p->pc3; + average.packages->pc3 += p->pc3; if (DO_BIC(BIC_Pkgpc6)) - average.packages.pc6 += p->pc6; + average.packages->pc6 += p->pc6; if (DO_BIC(BIC_Pkgpc7)) - average.packages.pc7 += p->pc7; - average.packages.pc8 += p->pc8; - average.packages.pc9 += p->pc9; - average.packages.pc10 += p->pc10; - average.packages.die_c6 += p->die_c6; + average.packages->pc7 += p->pc7; + average.packages->pc8 += p->pc8; + average.packages->pc9 += p->pc9; + average.packages->pc10 += p->pc10; + average.packages->die_c6 += p->die_c6; - average.packages.cpu_lpi = p->cpu_lpi; - average.packages.sys_lpi = p->sys_lpi; + average.packages->cpu_lpi = p->cpu_lpi; + average.packages->sys_lpi = p->sys_lpi; - rapl_counter_accumulate(&average.packages.energy_pkg, &p->energy_pkg); - rapl_counter_accumulate(&average.packages.energy_dram, &p->energy_dram); - rapl_counter_accumulate(&average.packages.energy_cores, &p->energy_cores); - rapl_counter_accumulate(&average.packages.energy_gfx, &p->energy_gfx); + rapl_counter_accumulate(&average.packages->energy_pkg, &p->energy_pkg); + rapl_counter_accumulate(&average.packages->energy_dram, &p->energy_dram); + rapl_counter_accumulate(&average.packages->energy_cores, &p->energy_cores); + rapl_counter_accumulate(&average.packages->energy_gfx, &p->energy_gfx); - average.packages.gfx_rc6_ms = p->gfx_rc6_ms; - average.packages.uncore_mhz = p->uncore_mhz; - average.packages.gfx_mhz = p->gfx_mhz; - average.packages.gfx_act_mhz = p->gfx_act_mhz; - average.packages.sam_mc6_ms = p->sam_mc6_ms; - average.packages.sam_mhz = p->sam_mhz; - average.packages.sam_act_mhz = p->sam_act_mhz; + average.packages->gfx_rc6_ms = p->gfx_rc6_ms; + average.packages->uncore_mhz = p->uncore_mhz; + average.packages->gfx_mhz = p->gfx_mhz; + average.packages->gfx_act_mhz = p->gfx_act_mhz; + average.packages->sam_mc6_ms = p->sam_mc6_ms; + average.packages->sam_mhz = p->sam_mhz; + average.packages->sam_act_mhz = p->sam_act_mhz; - average.packages.pkg_temp_c = MAX(average.packages.pkg_temp_c, p->pkg_temp_c); + average.packages->pkg_temp_c = MAX(average.packages->pkg_temp_c, p->pkg_temp_c); - rapl_counter_accumulate(&average.packages.rapl_pkg_perf_status, &p->rapl_pkg_perf_status); - rapl_counter_accumulate(&average.packages.rapl_dram_perf_status, &p->rapl_dram_perf_status); + rapl_counter_accumulate(&average.packages->rapl_pkg_perf_status, &p->rapl_pkg_perf_status); + rapl_counter_accumulate(&average.packages->rapl_dram_perf_status, &p->rapl_dram_perf_status); for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) { if ((mp->format == FORMAT_RAW) && (topo.num_packages == 0)) - average.packages.counter[i] = p->counter[i]; + average.packages->counter[i] = p->counter[i]; else - average.packages.counter[i] += p->counter[i]; + average.packages->counter[i] += p->counter[i]; } for (i = 0, pp = sys.perf_pp; pp; i++, pp = pp->next) { if ((pp->format == FORMAT_RAW) && (topo.num_packages == 0)) - average.packages.perf_counter[i] = p->perf_counter[i]; + average.packages->perf_counter[i] = p->perf_counter[i]; else - average.packages.perf_counter[i] += p->perf_counter[i]; + average.packages->perf_counter[i] += p->perf_counter[i]; } for (i = 0, ppmt = sys.pmt_pp; ppmt; i++, ppmt = ppmt->next) { - average.packages.pmt_counter[i] += p->pmt_counter[i]; + average.packages->pmt_counter[i] += p->pmt_counter[i]; } return 0; @@ -4167,117 +4318,117 @@ void compute_average(PER_THREAD_PARAMS) struct perf_counter_info *pp; struct pmt_counter *ppmt; - clear_counters(&average.threads, &average.cores, &average.packages); + clear_counters(average.threads, average.cores, average.packages); for_all_cpus(sum_counters, t, c, p); /* Use the global time delta for the average. */ - average.threads.tv_delta = tv_delta; + average.threads->tv_delta = tv_delta; - average.threads.tsc /= topo.allowed_cpus; - average.threads.aperf /= topo.allowed_cpus; - average.threads.mperf /= topo.allowed_cpus; - average.threads.instr_count /= topo.allowed_cpus; - average.threads.c1 /= topo.allowed_cpus; + average.threads->tsc /= topo.allowed_cpus; + average.threads->aperf /= topo.allowed_cpus; + average.threads->mperf /= topo.allowed_cpus; + average.threads->instr_count /= topo.allowed_cpus; + average.threads->c1 /= topo.allowed_cpus; - if (average.threads.irq_count > 9999999) + if (average.threads->irq_count > 9999999) sums_need_wide_columns = 1; - if (average.threads.nmi_count > 9999999) + if (average.threads->nmi_count > 9999999) sums_need_wide_columns = 1; - average.cores.c3 /= topo.allowed_cores; - average.cores.c6 /= topo.allowed_cores; - average.cores.c7 /= topo.allowed_cores; - average.cores.mc6_us /= topo.allowed_cores; + average.cores->c3 /= topo.allowed_cores; + average.cores->c6 /= topo.allowed_cores; + average.cores->c7 /= topo.allowed_cores; + average.cores->mc6_us /= topo.allowed_cores; if (DO_BIC(BIC_Totl_c0)) - average.packages.pkg_wtd_core_c0 /= topo.allowed_packages; + average.packages->pkg_wtd_core_c0 /= topo.allowed_packages; if (DO_BIC(BIC_Any_c0)) - average.packages.pkg_any_core_c0 /= topo.allowed_packages; + average.packages->pkg_any_core_c0 /= topo.allowed_packages; if (DO_BIC(BIC_GFX_c0)) - average.packages.pkg_any_gfxe_c0 /= topo.allowed_packages; + average.packages->pkg_any_gfxe_c0 /= topo.allowed_packages; if (DO_BIC(BIC_CPUGFX)) - average.packages.pkg_both_core_gfxe_c0 /= topo.allowed_packages; + average.packages->pkg_both_core_gfxe_c0 /= topo.allowed_packages; - average.packages.pc2 /= topo.allowed_packages; + average.packages->pc2 /= topo.allowed_packages; if (DO_BIC(BIC_Pkgpc3)) - average.packages.pc3 /= topo.allowed_packages; + average.packages->pc3 /= topo.allowed_packages; if (DO_BIC(BIC_Pkgpc6)) - average.packages.pc6 /= topo.allowed_packages; + average.packages->pc6 /= topo.allowed_packages; if (DO_BIC(BIC_Pkgpc7)) - average.packages.pc7 /= topo.allowed_packages; + average.packages->pc7 /= topo.allowed_packages; - average.packages.pc8 /= topo.allowed_packages; - average.packages.pc9 /= topo.allowed_packages; - average.packages.pc10 /= topo.allowed_packages; - average.packages.die_c6 /= topo.allowed_packages; + average.packages->pc8 /= topo.allowed_packages; + average.packages->pc9 /= topo.allowed_packages; + average.packages->pc10 /= topo.allowed_packages; + average.packages->die_c6 /= topo.allowed_packages; for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { if (mp->format == FORMAT_RAW) continue; if (mp->type == COUNTER_ITEMS) { - if (average.threads.counter[i] > 9999999) + if (average.threads->counter[i] > 9999999) sums_need_wide_columns = 1; continue; } - average.threads.counter[i] /= topo.allowed_cpus; + average.threads->counter[i] /= topo.allowed_cpus; } for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) { if (mp->format == FORMAT_RAW) continue; if (mp->type == COUNTER_ITEMS) { - if (average.cores.counter[i] > 9999999) + if (average.cores->counter[i] > 9999999) sums_need_wide_columns = 1; } - average.cores.counter[i] /= topo.allowed_cores; + average.cores->counter[i] /= topo.allowed_cores; } for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) { if (mp->format == FORMAT_RAW) continue; if (mp->type == COUNTER_ITEMS) { - if (average.packages.counter[i] > 9999999) + if (average.packages->counter[i] > 9999999) sums_need_wide_columns = 1; } - average.packages.counter[i] /= topo.allowed_packages; + average.packages->counter[i] /= topo.allowed_packages; } for (i = 0, pp = sys.perf_tp; pp; i++, pp = pp->next) { if (pp->format == FORMAT_RAW) continue; if (pp->type == COUNTER_ITEMS) { - if (average.threads.perf_counter[i] > 9999999) + if (average.threads->perf_counter[i] > 9999999) sums_need_wide_columns = 1; continue; } - average.threads.perf_counter[i] /= topo.allowed_cpus; + average.threads->perf_counter[i] /= topo.allowed_cpus; } for (i = 0, pp = sys.perf_cp; pp; i++, pp = pp->next) { if (pp->format == FORMAT_RAW) continue; if (pp->type == COUNTER_ITEMS) { - if (average.cores.perf_counter[i] > 9999999) + if (average.cores->perf_counter[i] > 9999999) sums_need_wide_columns = 1; } - average.cores.perf_counter[i] /= topo.allowed_cores; + average.cores->perf_counter[i] /= topo.allowed_cores; } for (i = 0, pp = sys.perf_pp; pp; i++, pp = pp->next) { if (pp->format == FORMAT_RAW) continue; if (pp->type == COUNTER_ITEMS) { - if (average.packages.perf_counter[i] > 9999999) + if (average.packages->perf_counter[i] > 9999999) sums_need_wide_columns = 1; } - average.packages.perf_counter[i] /= topo.allowed_packages; + average.packages->perf_counter[i] /= topo.allowed_packages; } for (i = 0, ppmt = sys.pmt_tp; ppmt; i++, ppmt = ppmt->next) { - average.threads.pmt_counter[i] /= topo.allowed_cpus; + average.threads->pmt_counter[i] /= topo.allowed_cpus; } for (i = 0, ppmt = sys.pmt_cp; ppmt; i++, ppmt = ppmt->next) { - average.cores.pmt_counter[i] /= topo.allowed_cores; + average.cores->pmt_counter[i] /= topo.allowed_cores; } for (i = 0, ppmt = sys.pmt_pp; ppmt; i++, ppmt = ppmt->next) { - average.packages.pmt_counter[i] /= topo.allowed_packages; + average.packages->pmt_counter[i] /= topo.allowed_packages; } } @@ -4645,7 +4796,7 @@ void write_rapl_counter(struct rapl_counter *rc, struct rapl_counter_info_t *rci int get_rapl_counters(int cpu, unsigned int domain, struct core_data *c, struct pkg_data *p) { - struct platform_counters *pplat_cnt = p == package_odd ? &platform_counters_odd : &platform_counters_even; + struct platform_counters *pplat_cnt = p == odd.packages ? &platform_counters_odd : &platform_counters_even; unsigned long long perf_data[NUM_RAPL_COUNTERS + 1]; struct rapl_counter_info_t *rci; @@ -5002,32 +5153,18 @@ unsigned long pmt_read_counter(struct pmt_counter *ppmt, unsigned int domain_id) /* Rapl domain enumeration helpers */ static inline int get_rapl_num_domains(void) { - int num_packages = topo.max_package_id + 1; - int num_cores_per_package; - int num_cores; - if (!platform->has_per_core_rapl) - return num_packages; + return topo.num_packages; - num_cores_per_package = topo.max_core_id + 1; - num_cores = num_cores_per_package * num_packages; - - return num_cores; + return topo.num_cores; } static inline int get_rapl_domain_id(int cpu) { - int nr_cores_per_package = topo.max_core_id + 1; - int rapl_core_id; - if (!platform->has_per_core_rapl) - return cpus[cpu].physical_package_id; + return cpus[cpu].package_id; - /* Compute the system-wide unique core-id for @cpu */ - rapl_core_id = cpus[cpu].physical_core_id; - rapl_core_id += cpus[cpu].physical_package_id * nr_cores_per_package; - - return rapl_core_id; + return GLOBAL_CORE_ID(cpus[cpu].core_id, cpus[cpu].package_id); } /* @@ -5058,9 +5195,12 @@ int get_counters(PER_THREAD_PARAMS) get_smi_aperf_mperf(cpu, t); - if (DO_BIC(BIC_LLC_RPS) || DO_BIC(BIC_LLC_HIT)) + if (DO_BIC(BIC_LLC_MRPS) || DO_BIC(BIC_LLC_HIT)) get_perf_llc_stats(cpu, &t->llc); + if (DO_BIC(BIC_L2_MRPS) || DO_BIC(BIC_L2_HIT)) + get_perf_l2_stats(cpu, &t->l2); + if (DO_BIC(BIC_IPC)) if (read(get_instr_count_fd(cpu), &t->instr_count, sizeof(long long)) != sizeof(long long)) return -4; @@ -5125,7 +5265,7 @@ int get_counters(PER_THREAD_PARAMS) return -10; for (i = 0, pp = sys.pmt_cp; pp; i++, pp = pp->next) - c->pmt_counter[i] = pmt_read_counter(pp, c->core_id); + c->pmt_counter[i] = pmt_read_counter(pp, cpus[t->cpu_id].core_id); /* collect package counters only for 1st core in package */ if (!is_cpu_first_core_in_package(t, p)) @@ -5166,7 +5306,7 @@ int get_counters(PER_THREAD_PARAMS) } if (DO_BIC(BIC_UNCORE_MHZ)) - p->uncore_mhz = get_legacy_uncore_mhz(p->package_id); + p->uncore_mhz = get_legacy_uncore_mhz(cpus[t->cpu_id].package_id); if (DO_BIC(BIC_GFX_rc6)) p->gfx_rc6_ms = gfx_info[GFX_rc6].val_ull; @@ -5190,9 +5330,9 @@ int get_counters(PER_THREAD_PARAMS) char *path = NULL; if (mp->msr_num == 0) { - path = find_sysfs_path_by_id(mp->sp, p->package_id); + path = find_sysfs_path_by_id(mp->sp, cpus[t->cpu_id].package_id); if (path == NULL) { - warnx("%s: package_id %d not found", __func__, p->package_id); + warnx("%s: package_id %d not found", __func__, cpus[t->cpu_id].package_id); return -10; } } @@ -5204,7 +5344,7 @@ int get_counters(PER_THREAD_PARAMS) return -10; for (i = 0, pp = sys.pmt_pp; pp; i++, pp = pp->next) - p->pmt_counter[i] = pmt_read_counter(pp, p->package_id); + p->pmt_counter[i] = pmt_read_counter(pp, cpus[t->cpu_id].package_id); done: gettimeofday(&t->tv_end, (struct timezone *)NULL); @@ -5293,7 +5433,7 @@ void probe_cst_limit(void) return; } - get_msr(base_cpu, MSR_PKG_CST_CONFIG_CONTROL, &msr); + get_msr(master_cpu, MSR_PKG_CST_CONFIG_CONTROL, &msr); pkg_cstate_limit = pkg_cstate_limits[msr & 0xF]; } @@ -5305,9 +5445,9 @@ static void dump_platform_info(void) if (!platform->has_nhm_msrs || no_msr) return; - get_msr(base_cpu, MSR_PLATFORM_INFO, &msr); + get_msr(master_cpu, MSR_PLATFORM_INFO, &msr); - fprintf(outf, "cpu%d: MSR_PLATFORM_INFO: 0x%08llx\n", base_cpu, msr); + fprintf(outf, "cpu%d: MSR_PLATFORM_INFO: 0x%08llx\n", master_cpu, msr); ratio = (msr >> 40) & 0xFF; fprintf(outf, "%d * %.1f = %.1f MHz max efficiency frequency\n", ratio, bclk, ratio * bclk); @@ -5323,8 +5463,8 @@ static void dump_power_ctl(void) if (!platform->has_nhm_msrs || no_msr) return; - get_msr(base_cpu, MSR_IA32_POWER_CTL, &msr); - fprintf(outf, "cpu%d: MSR_IA32_POWER_CTL: 0x%08llx (C1E auto-promotion: %sabled)\n", base_cpu, msr, msr & 0x2 ? "EN" : "DIS"); + get_msr(master_cpu, MSR_IA32_POWER_CTL, &msr); + fprintf(outf, "cpu%d: MSR_IA32_POWER_CTL: 0x%08llx (C1E auto-promotion: %sabled)\n", master_cpu, msr, msr & 0x2 ? "EN" : "DIS"); /* C-state Pre-wake Disable (CSTATE_PREWAKE_DISABLE) */ if (platform->has_cst_prewake_bit) @@ -5338,9 +5478,9 @@ static void dump_turbo_ratio_limit2(void) unsigned long long msr; unsigned int ratio; - get_msr(base_cpu, MSR_TURBO_RATIO_LIMIT2, &msr); + get_msr(master_cpu, MSR_TURBO_RATIO_LIMIT2, &msr); - fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT2: 0x%08llx\n", base_cpu, msr); + fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT2: 0x%08llx\n", master_cpu, msr); ratio = (msr >> 8) & 0xFF; if (ratio) @@ -5357,9 +5497,9 @@ static void dump_turbo_ratio_limit1(void) unsigned long long msr; unsigned int ratio; - get_msr(base_cpu, MSR_TURBO_RATIO_LIMIT1, &msr); + get_msr(master_cpu, MSR_TURBO_RATIO_LIMIT1, &msr); - fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT1: 0x%08llx\n", base_cpu, msr); + fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT1: 0x%08llx\n", master_cpu, msr); ratio = (msr >> 56) & 0xFF; if (ratio) @@ -5400,13 +5540,12 @@ static void dump_turbo_ratio_limits(int trl_msr_offset) unsigned long long msr, core_counts; int shift; - get_msr(base_cpu, trl_msr_offset, &msr); - fprintf(outf, "cpu%d: MSR_%sTURBO_RATIO_LIMIT: 0x%08llx\n", - base_cpu, trl_msr_offset == MSR_SECONDARY_TURBO_RATIO_LIMIT ? "SECONDARY_" : "", msr); + get_msr(master_cpu, trl_msr_offset, &msr); + fprintf(outf, "cpu%d: MSR_%sTURBO_RATIO_LIMIT: 0x%08llx\n", master_cpu, trl_msr_offset == MSR_SECONDARY_TURBO_RATIO_LIMIT ? "SECONDARY_" : "", msr); if (platform->trl_msrs & TRL_CORECOUNT) { - get_msr(base_cpu, MSR_TURBO_RATIO_LIMIT1, &core_counts); - fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT1: 0x%08llx\n", base_cpu, core_counts); + get_msr(master_cpu, MSR_TURBO_RATIO_LIMIT1, &core_counts); + fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT1: 0x%08llx\n", master_cpu, core_counts); } else { core_counts = 0x0807060504030201; } @@ -5428,8 +5567,8 @@ static void dump_atom_turbo_ratio_limits(void) unsigned long long msr; unsigned int ratio; - get_msr(base_cpu, MSR_ATOM_CORE_RATIOS, &msr); - fprintf(outf, "cpu%d: MSR_ATOM_CORE_RATIOS: 0x%08llx\n", base_cpu, msr & 0xFFFFFFFF); + get_msr(master_cpu, MSR_ATOM_CORE_RATIOS, &msr); + fprintf(outf, "cpu%d: MSR_ATOM_CORE_RATIOS: 0x%08llx\n", master_cpu, msr & 0xFFFFFFFF); ratio = (msr >> 0) & 0x3F; if (ratio) @@ -5443,8 +5582,8 @@ static void dump_atom_turbo_ratio_limits(void) if (ratio) fprintf(outf, "%d * %.1f = %.1f MHz base frequency\n", ratio, bclk, ratio * bclk); - get_msr(base_cpu, MSR_ATOM_CORE_TURBO_RATIOS, &msr); - fprintf(outf, "cpu%d: MSR_ATOM_CORE_TURBO_RATIOS: 0x%08llx\n", base_cpu, msr & 0xFFFFFFFF); + get_msr(master_cpu, MSR_ATOM_CORE_TURBO_RATIOS, &msr); + fprintf(outf, "cpu%d: MSR_ATOM_CORE_TURBO_RATIOS: 0x%08llx\n", master_cpu, msr & 0xFFFFFFFF); ratio = (msr >> 24) & 0x3F; if (ratio) @@ -5473,9 +5612,9 @@ static void dump_knl_turbo_ratio_limits(void) unsigned int cores[buckets_no]; unsigned int ratio[buckets_no]; - get_msr(base_cpu, MSR_TURBO_RATIO_LIMIT, &msr); + get_msr(master_cpu, MSR_TURBO_RATIO_LIMIT, &msr); - fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT: 0x%08llx\n", base_cpu, msr); + fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT: 0x%08llx\n", master_cpu, msr); /* * Turbo encoding in KNL is as follows: @@ -5525,9 +5664,9 @@ static void dump_cst_cfg(void) if (!platform->has_nhm_msrs || no_msr) return; - get_msr(base_cpu, MSR_PKG_CST_CONFIG_CONTROL, &msr); + get_msr(master_cpu, MSR_PKG_CST_CONFIG_CONTROL, &msr); - fprintf(outf, "cpu%d: MSR_PKG_CST_CONFIG_CONTROL: 0x%08llx", base_cpu, msr); + fprintf(outf, "cpu%d: MSR_PKG_CST_CONFIG_CONTROL: 0x%08llx", master_cpu, msr); fprintf(outf, " (%s%s%s%s%slocked, pkg-cstate-limit=%d (%s)", (msr & SNB_C3_AUTO_UNDEMOTE) ? "UNdemote-C3, " : "", @@ -5550,12 +5689,12 @@ static void dump_config_tdp(void) { unsigned long long msr; - get_msr(base_cpu, MSR_CONFIG_TDP_NOMINAL, &msr); - fprintf(outf, "cpu%d: MSR_CONFIG_TDP_NOMINAL: 0x%08llx", base_cpu, msr); + get_msr(master_cpu, MSR_CONFIG_TDP_NOMINAL, &msr); + fprintf(outf, "cpu%d: MSR_CONFIG_TDP_NOMINAL: 0x%08llx", master_cpu, msr); fprintf(outf, " (base_ratio=%d)\n", (unsigned int)msr & 0xFF); - get_msr(base_cpu, MSR_CONFIG_TDP_LEVEL_1, &msr); - fprintf(outf, "cpu%d: MSR_CONFIG_TDP_LEVEL_1: 0x%08llx (", base_cpu, msr); + get_msr(master_cpu, MSR_CONFIG_TDP_LEVEL_1, &msr); + fprintf(outf, "cpu%d: MSR_CONFIG_TDP_LEVEL_1: 0x%08llx (", master_cpu, msr); if (msr) { fprintf(outf, "PKG_MIN_PWR_LVL1=%d ", (unsigned int)(msr >> 48) & 0x7FFF); fprintf(outf, "PKG_MAX_PWR_LVL1=%d ", (unsigned int)(msr >> 32) & 0x7FFF); @@ -5564,8 +5703,8 @@ static void dump_config_tdp(void) } fprintf(outf, ")\n"); - get_msr(base_cpu, MSR_CONFIG_TDP_LEVEL_2, &msr); - fprintf(outf, "cpu%d: MSR_CONFIG_TDP_LEVEL_2: 0x%08llx (", base_cpu, msr); + get_msr(master_cpu, MSR_CONFIG_TDP_LEVEL_2, &msr); + fprintf(outf, "cpu%d: MSR_CONFIG_TDP_LEVEL_2: 0x%08llx (", master_cpu, msr); if (msr) { fprintf(outf, "PKG_MIN_PWR_LVL2=%d ", (unsigned int)(msr >> 48) & 0x7FFF); fprintf(outf, "PKG_MAX_PWR_LVL2=%d ", (unsigned int)(msr >> 32) & 0x7FFF); @@ -5574,15 +5713,15 @@ static void dump_config_tdp(void) } fprintf(outf, ")\n"); - get_msr(base_cpu, MSR_CONFIG_TDP_CONTROL, &msr); - fprintf(outf, "cpu%d: MSR_CONFIG_TDP_CONTROL: 0x%08llx (", base_cpu, msr); + get_msr(master_cpu, MSR_CONFIG_TDP_CONTROL, &msr); + fprintf(outf, "cpu%d: MSR_CONFIG_TDP_CONTROL: 0x%08llx (", master_cpu, msr); if ((msr) & 0x3) fprintf(outf, "TDP_LEVEL=%d ", (unsigned int)(msr) & 0x3); fprintf(outf, " lock=%d", (unsigned int)(msr >> 31) & 1); fprintf(outf, ")\n"); - get_msr(base_cpu, MSR_TURBO_ACTIVATION_RATIO, &msr); - fprintf(outf, "cpu%d: MSR_TURBO_ACTIVATION_RATIO: 0x%08llx (", base_cpu, msr); + get_msr(master_cpu, MSR_TURBO_ACTIVATION_RATIO, &msr); + fprintf(outf, "cpu%d: MSR_TURBO_ACTIVATION_RATIO: 0x%08llx (", master_cpu, msr); fprintf(outf, "MAX_NON_TURBO_RATIO=%d", (unsigned int)(msr) & 0xFF); fprintf(outf, " lock=%d", (unsigned int)(msr >> 31) & 1); fprintf(outf, ")\n"); @@ -5598,38 +5737,38 @@ void print_irtl(void) return; if (platform->supported_cstates & PC3) { - get_msr(base_cpu, MSR_PKGC3_IRTL, &msr); - fprintf(outf, "cpu%d: MSR_PKGC3_IRTL: 0x%08llx (", base_cpu, msr); + get_msr(master_cpu, MSR_PKGC3_IRTL, &msr); + fprintf(outf, "cpu%d: MSR_PKGC3_IRTL: 0x%08llx (", master_cpu, msr); fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); } if (platform->supported_cstates & PC6) { - get_msr(base_cpu, MSR_PKGC6_IRTL, &msr); - fprintf(outf, "cpu%d: MSR_PKGC6_IRTL: 0x%08llx (", base_cpu, msr); + get_msr(master_cpu, MSR_PKGC6_IRTL, &msr); + fprintf(outf, "cpu%d: MSR_PKGC6_IRTL: 0x%08llx (", master_cpu, msr); fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); } if (platform->supported_cstates & PC7) { - get_msr(base_cpu, MSR_PKGC7_IRTL, &msr); - fprintf(outf, "cpu%d: MSR_PKGC7_IRTL: 0x%08llx (", base_cpu, msr); + get_msr(master_cpu, MSR_PKGC7_IRTL, &msr); + fprintf(outf, "cpu%d: MSR_PKGC7_IRTL: 0x%08llx (", master_cpu, msr); fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); } if (platform->supported_cstates & PC8) { - get_msr(base_cpu, MSR_PKGC8_IRTL, &msr); - fprintf(outf, "cpu%d: MSR_PKGC8_IRTL: 0x%08llx (", base_cpu, msr); + get_msr(master_cpu, MSR_PKGC8_IRTL, &msr); + fprintf(outf, "cpu%d: MSR_PKGC8_IRTL: 0x%08llx (", master_cpu, msr); fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); } if (platform->supported_cstates & PC9) { - get_msr(base_cpu, MSR_PKGC9_IRTL, &msr); - fprintf(outf, "cpu%d: MSR_PKGC9_IRTL: 0x%08llx (", base_cpu, msr); + get_msr(master_cpu, MSR_PKGC9_IRTL, &msr); + fprintf(outf, "cpu%d: MSR_PKGC9_IRTL: 0x%08llx (", master_cpu, msr); fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); } if (platform->supported_cstates & PC10) { - get_msr(base_cpu, MSR_PKGC10_IRTL, &msr); - fprintf(outf, "cpu%d: MSR_PKGC10_IRTL: 0x%08llx (", base_cpu, msr); + get_msr(master_cpu, MSR_PKGC10_IRTL, &msr); + fprintf(outf, "cpu%d: MSR_PKGC10_IRTL: 0x%08llx (", master_cpu, msr); fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); } } @@ -5676,6 +5815,26 @@ void free_fd_llc_percpu(void) free(fd_llc_percpu); fd_llc_percpu = NULL; + + BIC_NOT_PRESENT(BIC_LLC_MRPS); + BIC_NOT_PRESENT(BIC_LLC_HIT); +} + +void free_fd_l2_percpu(void) +{ + if (!fd_l2_percpu) + return; + + for (int i = 0; i < topo.max_cpu_num + 1; ++i) { + if (fd_l2_percpu[i] != 0) + close(fd_l2_percpu[i]); + } + + free(fd_l2_percpu); + fd_l2_percpu = NULL; + + BIC_NOT_PRESENT(BIC_L2_MRPS); + BIC_NOT_PRESENT(BIC_L2_HIT); } void free_fd_cstate(void) @@ -5780,21 +5939,36 @@ void free_all_buffers(void) cpu_affinity_set = NULL; cpu_affinity_setsize = 0; - free(thread_even); - free(core_even); - free(package_even); + if (perf_pcore_set) { + CPU_FREE(perf_pcore_set); + perf_pcore_set = NULL; + } - thread_even = NULL; - core_even = NULL; - package_even = NULL; + if (perf_ecore_set) { + CPU_FREE(perf_ecore_set); + perf_ecore_set = NULL; + } - free(thread_odd); - free(core_odd); - free(package_odd); + if (perf_lcore_set) { + CPU_FREE(perf_lcore_set); + perf_lcore_set = NULL; + } - thread_odd = NULL; - core_odd = NULL; - package_odd = NULL; + free(even.threads); + free(even.cores); + free(even.packages); + + even.threads = NULL; + even.cores = NULL; + even.packages = NULL; + + free(odd.threads); + free(odd.cores); + free(odd.packages); + + odd.threads = NULL; + odd.cores = NULL; + odd.packages = NULL; free(output_buffer); output_buffer = NULL; @@ -5803,6 +5977,7 @@ void free_all_buffers(void) free_fd_percpu(); free_fd_instr_count_percpu(); free_fd_llc_percpu(); + free_fd_l2_percpu(); free_fd_msr(); free_fd_rapl_percpu(); free_fd_cstate(); @@ -5852,7 +6027,7 @@ int cpu_is_first_core_in_package(int cpu) return cpu == parse_int_file("/sys/devices/system/cpu/cpu%d/topology/core_siblings_list", cpu); } -int get_physical_package_id(int cpu) +int get_package_id(int cpu) { return parse_int_file("/sys/devices/system/cpu/cpu%d/topology/physical_package_id", cpu); } @@ -5885,7 +6060,7 @@ void set_node_data(void) for (pkg = 0; pkg < topo.num_packages; pkg++) { lnode = 0; for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu) { - if (cpus[cpu].physical_package_id != pkg) + if (cpus[cpu].package_id != pkg) continue; /* find a cpu with an unset logical_node_id */ if (cpus[cpu].logical_node_id != -1) @@ -5898,7 +6073,7 @@ void set_node_data(void) * the logical_node_id */ for (cpux = cpu; cpux <= topo.max_cpu_num; cpux++) { - if ((cpus[cpux].physical_package_id == pkg) && (cpus[cpux].physical_node_id == node)) { + if ((cpus[cpux].package_id == pkg) && (cpus[cpux].physical_node_id == node)) { cpus[cpux].logical_node_id = lnode; cpu_count++; } @@ -5917,7 +6092,7 @@ int get_physical_node_id(struct cpu_topology *thiscpu) char path[80]; FILE *filep; int i; - int cpu = thiscpu->logical_cpu_id; + int cpu = thiscpu->cpu_id; for (i = 0; i <= topo.max_cpu_num; i++) { sprintf(path, "/sys/devices/system/cpu/cpu%d/node%i/cpulist", cpu, i); @@ -5986,20 +6161,20 @@ static int parse_cpu_str(char *cpu_str, cpu_set_t *cpu_set, int cpu_set_size) return 0; } -int get_thread_siblings(struct cpu_topology *thiscpu) +int set_thread_siblings(struct cpu_topology *thiscpu) { char path[80], character; FILE *filep; unsigned long map; int so, shift, sib_core; - int cpu = thiscpu->logical_cpu_id; + int cpu = thiscpu->cpu_id; int offset = topo.max_cpu_num + 1; size_t size; int thread_id = 0; thiscpu->put_ids = CPU_ALLOC((topo.max_cpu_num + 1)); - if (thiscpu->thread_id < 0) - thiscpu->thread_id = thread_id++; + if (thiscpu->ht_id < 0) + thiscpu->ht_id = thread_id++; if (!thiscpu->put_ids) return -1; @@ -6021,10 +6196,15 @@ int get_thread_siblings(struct cpu_topology *thiscpu) if ((map >> shift) & 0x1) { so = shift + offset; sib_core = get_core_id(so); - if (sib_core == thiscpu->physical_core_id) { + if (sib_core == thiscpu->core_id) { CPU_SET_S(so, size, thiscpu->put_ids); - if ((so != cpu) && (cpus[so].thread_id < 0)) - cpus[so].thread_id = thread_id++; + if ((so != cpu) && (cpus[so].ht_id < 0)) { + cpus[so].ht_id = thread_id; + cpus[cpu].ht_sibling_cpu_id[thread_id] = so; + if (debug) + fprintf(stderr, "%s: cpu%d.ht_sibling_cpu_id[%d] = %d\n", __func__, cpu, thread_id, so); + thread_id += 1; + } } } } @@ -6045,30 +6225,40 @@ int for_all_cpus_2(int (func) (struct thread_data *, struct core_data *, struct core_data *core_base, struct pkg_data *pkg_base, struct thread_data *thread_base2, struct core_data *core_base2, struct pkg_data *pkg_base2) { - int retval, pkg_no, node_no, core_no, thread_no; + int cpu, retval; retval = 0; - for (pkg_no = 0; pkg_no < topo.num_packages; ++pkg_no) { - for (node_no = 0; node_no < topo.nodes_per_pkg; ++node_no) { - for (core_no = 0; core_no < topo.cores_per_node; ++core_no) { - for (thread_no = 0; thread_no < topo.threads_per_core; ++thread_no) { - struct thread_data *t, *t2; - struct core_data *c, *c2; + for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu) { + struct thread_data *t, *t2; + struct core_data *c, *c2; + struct pkg_data *p, *p2; - t = GET_THREAD(thread_base, thread_no, core_no, node_no, pkg_no); + if (cpu_is_not_allowed(cpu)) + continue; - if (cpu_is_not_allowed(t->cpu_id)) - continue; + if (cpus[cpu].ht_id > 0) /* skip HT sibling */ + continue; - t2 = GET_THREAD(thread_base2, thread_no, core_no, node_no, pkg_no); + t = &thread_base[cpu]; + t2 = &thread_base2[cpu]; + c = &core_base[GLOBAL_CORE_ID(cpus[cpu].core_id, cpus[cpu].package_id)]; + c2 = &core_base2[GLOBAL_CORE_ID(cpus[cpu].core_id, cpus[cpu].package_id)]; + p = &pkg_base[cpus[cpu].package_id]; + p2 = &pkg_base2[cpus[cpu].package_id]; - c = GET_CORE(core_base, core_no, node_no, pkg_no); - c2 = GET_CORE(core_base2, core_no, node_no, pkg_no); + retval |= func(t, c, p, t2, c2, p2); - retval |= func(t, c, &pkg_base[pkg_no], t2, c2, &pkg_base2[pkg_no]); - } - } + /* Handle HT sibling now */ + int i; + + for (i = MAX_HT_ID; i > 0; --i) { /* ht_id 0 is self */ + if (cpus[cpu].ht_sibling_cpu_id[i] <= 0) + continue; + t = &thread_base[cpus[cpu].ht_sibling_cpu_id[i]]; + t2 = &thread_base2[cpus[cpu].ht_sibling_cpu_id[i]]; + + retval |= func(t, c, p, t2, c2, p2); } } return retval; @@ -6125,7 +6315,7 @@ static int update_effective_str(bool startup) pos = fgets(buf, 1024, fp); if (!pos) - err(1, "%s: file read failed\n", PATH_EFFECTIVE_CPUS); + err(1, "%s: file read failed", PATH_EFFECTIVE_CPUS); fclose(fp); @@ -6142,7 +6332,7 @@ static void update_effective_set(bool startup) update_effective_str(startup); if (parse_cpu_str(cpu_effective_str, cpu_effective_set, cpu_effective_setsize)) - err(1, "%s: cpu str malformat %s\n", PATH_EFFECTIVE_CPUS, cpu_effective_str); + err(1, "%s: cpu str malformat %s", PATH_EFFECTIVE_CPUS, cpu_effective_str); } void linux_perf_init(void); @@ -6150,6 +6340,7 @@ void msr_perf_init(void); void rapl_perf_init(void); void cstate_perf_init(void); void perf_llc_init(void); +void perf_l2_init(void); void added_perf_counters_init(void); void pmt_init(void); @@ -6162,6 +6353,7 @@ void re_initialize(void) rapl_perf_init(); cstate_perf_init(); perf_llc_init(); + perf_l2_init(); added_perf_counters_init(); pmt_init(); fprintf(outf, "turbostat: re-initialized with num_cpus %d, allowed_cpus %d\n", topo.num_cpus, topo.allowed_cpus); @@ -6170,14 +6362,14 @@ void re_initialize(void) void set_max_cpu_num(void) { FILE *filep; - int base_cpu; + int current_cpu; unsigned long dummy; char pathname[64]; - base_cpu = sched_getcpu(); - if (base_cpu < 0) + current_cpu = sched_getcpu(); + if (current_cpu < 0) err(1, "cannot find calling cpu ID"); - sprintf(pathname, "/sys/devices/system/cpu/cpu%d/topology/thread_siblings", base_cpu); + sprintf(pathname, "/sys/devices/system/cpu/cpu%d/topology/thread_siblings", current_cpu); filep = fopen_or_die(pathname, "r"); topo.max_cpu_num = 0; @@ -6205,9 +6397,13 @@ int mark_cpu_present(int cpu) return 0; } -int init_thread_id(int cpu) +int clear_ht_id(int cpu) { - cpus[cpu].thread_id = -1; + int i; + + cpus[cpu].ht_id = -1; + for (i = 0; i <= MAX_HT_ID; ++i) + cpus[cpu].ht_sibling_cpu_id[i] = -1; return 0; } @@ -6740,7 +6936,7 @@ int probe_dev_msr(void) struct stat sb; char pathname[32]; - sprintf(pathname, "/dev/msr%d", base_cpu); + sprintf(pathname, "/dev/msr%d", master_cpu); return !stat(pathname, &sb); } @@ -6749,7 +6945,7 @@ int probe_dev_cpu_msr(void) struct stat sb; char pathname[32]; - sprintf(pathname, "/dev/cpu/%d/msr", base_cpu); + sprintf(pathname, "/dev/cpu/%d/msr", master_cpu); return !stat(pathname, &sb); } @@ -6809,7 +7005,7 @@ int check_for_cap_sys_rawio(void) free_and_exit: if (cap_free(caps) == -1) - err(-6, "cap_free\n"); + err(-6, "cap_free"); return ret; } @@ -6826,7 +7022,7 @@ void check_msr_permission(void) failed += check_for_cap_sys_rawio(); /* test file permissions */ - sprintf(pathname, use_android_msr_path ? "/dev/msr%d" : "/dev/cpu/%d/msr", base_cpu); + sprintf(pathname, use_android_msr_path ? "/dev/msr%d" : "/dev/cpu/%d/msr", master_cpu); if (euidaccess(pathname, R_OK)) { failed++; } @@ -6855,7 +7051,7 @@ void probe_bclk(void) else return; - get_msr(base_cpu, MSR_PLATFORM_INFO, &msr); + get_msr(master_cpu, MSR_PLATFORM_INFO, &msr); base_ratio = (msr >> 8) & 0xFF; base_hz = base_ratio * bclk * 1000000; @@ -7006,16 +7202,16 @@ static void probe_intel_uncore_frequency_cluster(void) } for (i = uncore_max_id; i >= 0; --i) { int k, l; - int package_id, domain_id, cluster_id; + int unc_pkg_id, domain_id, cluster_id; char name_buf[16]; sprintf(path_base, "/sys/devices/system/cpu/intel_uncore_frequency/uncore%02d", i); if (access(path_base, R_OK)) - err(1, "%s: %s\n", __func__, path_base); + err(1, "%s: %s", __func__, path_base); sprintf(path, "%s/package_id", path_base); - package_id = read_sysfs_int(path); + unc_pkg_id = read_sysfs_int(path); sprintf(path, "%s/domain_id", path_base); domain_id = read_sysfs_int(path); @@ -7038,7 +7234,7 @@ static void probe_intel_uncore_frequency_cluster(void) */ if BIC_IS_ENABLED (BIC_UNCORE_MHZ) - add_counter(0, path, name_buf, 0, SCOPE_PACKAGE, COUNTER_K2M, FORMAT_AVERAGE, 0, package_id); + add_counter(0, path, name_buf, 0, SCOPE_PACKAGE, COUNTER_K2M, FORMAT_AVERAGE, 0, unc_pkg_id); if (quiet) continue; @@ -7047,7 +7243,7 @@ static void probe_intel_uncore_frequency_cluster(void) k = read_sysfs_int(path); sprintf(path, "%s/max_freq_khz", path_base); l = read_sysfs_int(path); - fprintf(outf, "Uncore Frequency package%d domain%d cluster%d: %d - %d MHz ", package_id, domain_id, cluster_id, k / 1000, l / 1000); + fprintf(outf, "Uncore Frequency package%d domain%d cluster%d: %d - %d MHz ", unc_pkg_id, domain_id, cluster_id, k / 1000, l / 1000); sprintf(path, "%s/initial_min_freq_khz", path_base); k = read_sysfs_int(path); @@ -7202,7 +7398,7 @@ static void dump_sysfs_cstate_config(void) for (state = 0; state < 10; ++state) { - sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name", base_cpu, state); + sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name", master_cpu, state); input = fopen(path, "r"); if (input == NULL) continue; @@ -7218,14 +7414,14 @@ static void dump_sysfs_cstate_config(void) remove_underbar(name_buf); - sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/desc", base_cpu, state); + sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/desc", master_cpu, state); input = fopen(path, "r"); if (input == NULL) continue; if (!fgets(desc, sizeof(desc), input)) err(1, "%s: failed to read file", path); - fprintf(outf, "cpu%d: %s: %s", base_cpu, name_buf, desc); + fprintf(outf, "cpu%d: %s: %s", master_cpu, name_buf, desc); fclose(input); } } @@ -7238,7 +7434,7 @@ static void dump_sysfs_pstate_config(void) FILE *input; int turbo; - sprintf(path, "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_driver", base_cpu); + sprintf(path, "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_driver", master_cpu); input = fopen(path, "r"); if (input == NULL) { fprintf(outf, "NSFOD %s\n", path); @@ -7248,7 +7444,7 @@ static void dump_sysfs_pstate_config(void) err(1, "%s: failed to read file", path); fclose(input); - sprintf(path, "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_governor", base_cpu); + sprintf(path, "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_governor", master_cpu); input = fopen(path, "r"); if (input == NULL) { fprintf(outf, "NSFOD %s\n", path); @@ -7258,8 +7454,8 @@ static void dump_sysfs_pstate_config(void) err(1, "%s: failed to read file", path); fclose(input); - fprintf(outf, "cpu%d: cpufreq driver: %s", base_cpu, driver_buf); - fprintf(outf, "cpu%d: cpufreq governor: %s", base_cpu, governor_buf); + fprintf(outf, "cpu%d: cpufreq driver: %s", master_cpu, driver_buf); + fprintf(outf, "cpu%d: cpufreq governor: %s", master_cpu, governor_buf); sprintf(path, "/sys/devices/system/cpu/cpufreq/boost"); input = fopen(path, "r"); @@ -7521,7 +7717,7 @@ double get_tdp_intel(void) unsigned long long msr; if (valid_rapl_msrs & RAPL_PKG_POWER_INFO) - if (!get_msr(base_cpu, MSR_PKG_POWER_INFO, &msr)) + if (!get_msr(master_cpu, MSR_PKG_POWER_INFO, &msr)) return ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units; return get_quirk_tdp(); } @@ -7560,7 +7756,7 @@ void rapl_probe_intel(void) CLR_BIC(BIC_RAM__, &bic_enabled); /* units on package 0, verify later other packages match */ - if (get_msr(base_cpu, MSR_RAPL_POWER_UNIT, &msr)) + if (get_msr(master_cpu, MSR_RAPL_POWER_UNIT, &msr)) return; rapl_power_units = 1.0 / (1 << (msr & 0xF)); @@ -7608,7 +7804,7 @@ void rapl_probe_amd(void) if (!valid_rapl_msrs || no_msr) return; - if (get_msr(base_cpu, MSR_RAPL_PWR_UNIT, &msr)) + if (get_msr(master_cpu, MSR_RAPL_PWR_UNIT, &msr)) return; rapl_time_units = ldexp(1.0, -(msr >> 16 & 0xf)); @@ -7817,8 +8013,7 @@ int print_rapl(PER_THREAD_PARAMS) return -1; } - fprintf(outf, "cpu%d: %s: 0x%08llx (%f Watts, %f Joules, %f sec.)\n", cpu, msr_name, msr, - rapl_power_units, rapl_energy_units, rapl_time_units); + fprintf(outf, "cpu%d: %s: 0x%08llx (%f Watts, %f Joules, %f sec.)\n", cpu, msr_name, msr, rapl_power_units, rapl_energy_units, rapl_time_units); if (valid_rapl_msrs & RAPL_PKG_POWER_INFO) { @@ -7850,8 +8045,7 @@ int print_rapl(PER_THREAD_PARAMS) return -9; fprintf(outf, "cpu%d: MSR_VR_CURRENT_CONFIG: 0x%08llx\n", cpu, msr); - fprintf(outf, "cpu%d: PKG Limit #4: %f Watts (%slocked)\n", - cpu, ((msr >> 0) & 0x1FFF) * rapl_power_units, (msr >> 31) & 1 ? "" : "UN"); + fprintf(outf, "cpu%d: PKG Limit #4: %f Watts (%slocked)\n", cpu, ((msr >> 0) & 0x1FFF) * rapl_power_units, (msr >> 31) & 1 ? "" : "UN"); } if (valid_rapl_msrs & RAPL_DRAM_POWER_INFO) { @@ -7919,7 +8113,7 @@ void probe_rapl_msrs(void) if (offset < 0) return; - ret = get_msr(base_cpu, offset, &msr_value); + ret = get_msr(master_cpu, offset, &msr_value); if (ret) { if (debug) fprintf(outf, "Can not read RAPL_PKG_ENERGY MSR(0x%llx)\n", (unsigned long long)offset); @@ -8004,7 +8198,7 @@ int set_temperature_target(PER_THREAD_PARAMS) if (!platform->has_nhm_msrs || no_msr) goto guess; - if (get_msr(base_cpu, MSR_IA32_TEMPERATURE_TARGET, &msr)) + if (get_msr(master_cpu, MSR_IA32_TEMPERATURE_TARGET, &msr)) goto guess; tcc_default = (msr >> 16) & 0xFF; @@ -8013,7 +8207,7 @@ int set_temperature_target(PER_THREAD_PARAMS) int bits = platform->tcc_offset_bits; unsigned long long enabled = 0; - if (bits && !get_msr(base_cpu, MSR_PLATFORM_INFO, &enabled)) + if (bits && !get_msr(master_cpu, MSR_PLATFORM_INFO, &enabled)) enabled = (enabled >> 30) & 1; if (bits && enabled) { @@ -8148,9 +8342,12 @@ void decode_feature_control_msr(void) if (no_msr) return; - if (!get_msr(base_cpu, MSR_IA32_FEAT_CTL, &msr)) + if (quiet) + return; + + if (!get_msr(master_cpu, MSR_IA32_FEAT_CTL, &msr)) fprintf(outf, "cpu%d: MSR_IA32_FEATURE_CONTROL: 0x%08llx (%sLocked %s)\n", - base_cpu, msr, msr & FEAT_CTL_LOCKED ? "" : "UN-", msr & (1 << 18) ? "SGX" : ""); + master_cpu, msr, msr & FEAT_CTL_LOCKED ? "" : "UN-", msr & (1 << 18) ? "SGX" : ""); } void decode_misc_enable_msr(void) @@ -8163,9 +8360,9 @@ void decode_misc_enable_msr(void) if (!genuine_intel) return; - if (!get_msr(base_cpu, MSR_IA32_MISC_ENABLE, &msr)) + if (!get_msr(master_cpu, MSR_IA32_MISC_ENABLE, &msr)) fprintf(outf, "cpu%d: MSR_IA32_MISC_ENABLE: 0x%08llx (%sTCC %sEIST %sMWAIT %sPREFETCH %sTURBO)\n", - base_cpu, msr, + master_cpu, msr, msr & MSR_IA32_MISC_ENABLE_TM1 ? "" : "No-", msr & MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP ? "" : "No-", msr & MSR_IA32_MISC_ENABLE_MWAIT ? "" : "No-", @@ -8182,11 +8379,10 @@ void decode_misc_feature_control(void) if (!platform->has_msr_misc_feature_control) return; - if (!get_msr(base_cpu, MSR_MISC_FEATURE_CONTROL, &msr)) + if (!get_msr(master_cpu, MSR_MISC_FEATURE_CONTROL, &msr)) fprintf(outf, "cpu%d: MSR_MISC_FEATURE_CONTROL: 0x%08llx (%sL2-Prefetch %sL2-Prefetch-pair %sL1-Prefetch %sL1-IP-Prefetch)\n", - base_cpu, msr, msr & (0 << 0) ? "No-" : "", msr & (1 << 0) ? "No-" : "", - msr & (2 << 0) ? "No-" : "", msr & (3 << 0) ? "No-" : ""); + master_cpu, msr, msr & (0 << 0) ? "No-" : "", msr & (1 << 0) ? "No-" : "", msr & (2 << 0) ? "No-" : "", msr & (3 << 0) ? "No-" : ""); } /* @@ -8206,9 +8402,9 @@ void decode_misc_pwr_mgmt_msr(void) if (!platform->has_msr_misc_pwr_mgmt) return; - if (!get_msr(base_cpu, MSR_MISC_PWR_MGMT, &msr)) + if (!get_msr(master_cpu, MSR_MISC_PWR_MGMT, &msr)) fprintf(outf, "cpu%d: MSR_MISC_PWR_MGMT: 0x%08llx (%sable-EIST_Coordination %sable-EPB %sable-OOB)\n", - base_cpu, msr, msr & (1 << 0) ? "DIS" : "EN", msr & (1 << 1) ? "EN" : "DIS", msr & (1 << 8) ? "EN" : "DIS"); + master_cpu, msr, msr & (1 << 0) ? "DIS" : "EN", msr & (1 << 1) ? "EN" : "DIS", msr & (1 << 8) ? "EN" : "DIS"); } /* @@ -8227,13 +8423,11 @@ void decode_c6_demotion_policy_msr(void) if (!platform->has_msr_c6_demotion_policy_config) return; - if (!get_msr(base_cpu, MSR_CC6_DEMOTION_POLICY_CONFIG, &msr)) - fprintf(outf, "cpu%d: MSR_CC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-CC6-Demotion)\n", - base_cpu, msr, msr & (1 << 0) ? "EN" : "DIS"); + if (!get_msr(master_cpu, MSR_CC6_DEMOTION_POLICY_CONFIG, &msr)) + fprintf(outf, "cpu%d: MSR_CC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-CC6-Demotion)\n", master_cpu, msr, msr & (1 << 0) ? "EN" : "DIS"); - if (!get_msr(base_cpu, MSR_MC6_DEMOTION_POLICY_CONFIG, &msr)) - fprintf(outf, "cpu%d: MSR_MC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-MC6-Demotion)\n", - base_cpu, msr, msr & (1 << 0) ? "EN" : "DIS"); + if (!get_msr(master_cpu, MSR_MC6_DEMOTION_POLICY_CONFIG, &msr)) + fprintf(outf, "cpu%d: MSR_MC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-MC6-Demotion)\n", master_cpu, msr, msr & (1 << 0) ? "EN" : "DIS"); } void print_dev_latency(void) @@ -8268,7 +8462,7 @@ static int has_perf_instr_count_access(void) if (no_perf) return 0; - fd = open_perf_counter(base_cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_INSTRUCTIONS, -1, 0); + fd = open_perf_counter(master_cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_INSTRUCTIONS, -1, 0); if (fd != -1) close(fd); @@ -8321,25 +8515,126 @@ end: return ret; } +char cpuset_buf[1024]; +int initialize_cpu_set_from_sysfs(cpu_set_t *cpu_set, char *sysfs_path, char *sysfs_file) +{ + FILE *fp; + char path[128]; + + if (snprintf(path, 128, "%s/%s", sysfs_path, sysfs_file) > 128) + err(-1, "%s %s", sysfs_path, sysfs_file); + + fp = fopen(path, "r"); + if (!fp) { + warn("open %s", path); + return -1; + } + if (fread(cpuset_buf, sizeof(char), 1024, fp) == 0) { + warn("read %s", sysfs_path); + goto err; + } + if (parse_cpu_str(cpuset_buf, cpu_set, cpu_possible_setsize)) { + warnx("%s: cpu str malformat %s\n", sysfs_path, cpu_effective_str); + goto err; + } + return 0; + +err: + fclose(fp); + return -1; +} + +void print_cpu_set(char *s, cpu_set_t *set) +{ + int i; + + assert(MAX_BIC < CPU_SETSIZE); + + printf("%s:", s); + + for (i = 0; i <= topo.max_cpu_num; ++i) + if (CPU_ISSET(i, set)) + printf(" %d", i); + putchar('\n'); +} + +void linux_perf_init_hybrid_cpus(void) +{ + char *perf_cpu_pcore_path = "/sys/devices/cpu_core"; + char *perf_cpu_ecore_path = "/sys/devices/cpu_atom"; + char *perf_cpu_lcore_path = "/sys/devices/cpu_lowpower"; + char path[128]; + + if (!access(perf_cpu_pcore_path, F_OK)) { + perf_pcore_set = CPU_ALLOC((topo.max_cpu_num + 1)); + if (perf_pcore_set == NULL) + err(3, "CPU_ALLOC"); + CPU_ZERO_S(cpu_possible_setsize, perf_pcore_set); + initialize_cpu_set_from_sysfs(perf_pcore_set, perf_cpu_pcore_path, "cpus"); + if (debug) + print_cpu_set("perf pcores", perf_pcore_set); + sprintf(path, "%s/%s", perf_cpu_pcore_path, "type"); + perf_pmu_types.pcore = snapshot_sysfs_counter(path); + } + + if (!access(perf_cpu_ecore_path, F_OK)) { + perf_ecore_set = CPU_ALLOC((topo.max_cpu_num + 1)); + if (perf_ecore_set == NULL) + err(3, "CPU_ALLOC"); + CPU_ZERO_S(cpu_possible_setsize, perf_ecore_set); + initialize_cpu_set_from_sysfs(perf_ecore_set, perf_cpu_ecore_path, "cpus"); + if (debug) + print_cpu_set("perf ecores", perf_ecore_set); + sprintf(path, "%s/%s", perf_cpu_ecore_path, "type"); + perf_pmu_types.ecore = snapshot_sysfs_counter(path); + } + + if (!access(perf_cpu_lcore_path, F_OK)) { + perf_lcore_set = CPU_ALLOC((topo.max_cpu_num + 1)); + if (perf_lcore_set == NULL) + err(3, "CPU_ALLOC"); + CPU_ZERO_S(cpu_possible_setsize, perf_lcore_set); + initialize_cpu_set_from_sysfs(perf_lcore_set, perf_cpu_lcore_path, "cpus"); + if (debug) + print_cpu_set("perf lcores", perf_lcore_set); + sprintf(path, "%s/%s", perf_cpu_lcore_path, "type"); + perf_pmu_types.lcore = snapshot_sysfs_counter(path); + } +} + /* - * Linux-perf manages the HW instructions-retired counter - * by enabling when requested, and hiding rollover + * Linux-perf related initialization */ void linux_perf_init(void) { + char path[128]; + char *perf_cpu_path = "/sys/devices/cpu"; + if (access("/proc/sys/kernel/perf_event_paranoid", F_OK)) return; + if (!access(perf_cpu_path, F_OK)) { + sprintf(path, "%s/%s", perf_cpu_path, "type"); + perf_pmu_types.uniform = snapshot_sysfs_counter(path); + } else { + linux_perf_init_hybrid_cpus(); + } + if (BIC_IS_ENABLED(BIC_IPC) && cpuid_has_aperf_mperf) { fd_instr_count_percpu = calloc(topo.max_cpu_num + 1, sizeof(int)); if (fd_instr_count_percpu == NULL) err(-1, "calloc fd_instr_count_percpu"); } - if (BIC_IS_ENABLED(BIC_LLC_RPS)) { + if (BIC_IS_ENABLED(BIC_LLC_MRPS) || BIC_IS_ENABLED(BIC_LLC_HIT)) { fd_llc_percpu = calloc(topo.max_cpu_num + 1, sizeof(int)); if (fd_llc_percpu == NULL) err(-1, "calloc fd_llc_percpu"); } + if (BIC_IS_ENABLED(BIC_L2_MRPS) || BIC_IS_ENABLED(BIC_L2_HIT)) { + fd_l2_percpu = calloc(topo.max_cpu_num + 1, sizeof(int)); + if (fd_l2_percpu == NULL) + err(-1, "calloc fd_l2_percpu"); + } } void rapl_perf_init(void) @@ -8397,7 +8692,7 @@ void rapl_perf_init(void) domain_visited[next_domain] = 1; - if ((cai->flags & RAPL_COUNTER_FLAG_PLATFORM_COUNTER) && (cpu != base_cpu)) + if ((cai->flags & RAPL_COUNTER_FLAG_PLATFORM_COUNTER) && (cpu != master_cpu)) continue; struct rapl_counter_info_t *rci = &rapl_counter_info_perdomain[next_domain]; @@ -8450,8 +8745,7 @@ void rapl_perf_init(void) /* Assumes msr_counter_info is populated */ static int has_amperf_access(void) { - return cpuid_has_aperf_mperf && msr_counter_arch_infos[MSR_ARCH_INFO_APERF_INDEX].present && - msr_counter_arch_infos[MSR_ARCH_INFO_MPERF_INDEX].present; + return cpuid_has_aperf_mperf && msr_counter_arch_infos[MSR_ARCH_INFO_APERF_INDEX].present && msr_counter_arch_infos[MSR_ARCH_INFO_MPERF_INDEX].present; } int *get_cstate_perf_group_fd(struct cstate_counter_info_t *cci, const char *group_name) @@ -8647,8 +8941,8 @@ void cstate_perf_init_(bool soft_c1) if (cpu_is_not_allowed(cpu)) continue; - const int core_id = cpus[cpu].physical_core_id; - const int pkg_id = cpus[cpu].physical_package_id; + const int core_id = cpus[cpu].core_id; + const int pkg_id = cpus[cpu].package_id; assert(core_id < cores_visited_elems); assert(pkg_id < pkg_visited_elems); @@ -8662,8 +8956,7 @@ void cstate_perf_init_(bool soft_c1) if (!per_core && pkg_visited[pkg_id]) continue; - const bool counter_needed = BIC_IS_ENABLED(cai->bic_number) || - (soft_c1 && (cai->flags & CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY)); + const bool counter_needed = BIC_IS_ENABLED(cai->bic_number) || (soft_c1 && (cai->flags & CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY)); const bool counter_supported = (platform->supported_cstates & cai->feature_mask); if (counter_needed && counter_supported) { @@ -8772,6 +9065,29 @@ void probe_pstates(void) for_all_cpus(print_perf_limit, ODD_COUNTERS); } +void dump_word_chars(unsigned int word) +{ + int i; + + for (i = 0; i < 4; ++i) + fprintf(outf, "%c", (word >> (i * 8)) & 0xFF); +} + +void dump_cpuid_hypervisor(void) +{ + unsigned int ebx = 0; + unsigned int ecx = 0; + unsigned int edx = 0; + + __cpuid(0x40000000, max_extended_level, ebx, ecx, edx); + + fprintf(outf, "Hypervisor: "); + dump_word_chars(ebx); + dump_word_chars(ecx); + dump_word_chars(edx); + fprintf(outf, "\n"); +} + void process_cpuid() { unsigned int eax, ebx, ecx, edx; @@ -8803,6 +9119,7 @@ void process_cpuid() model += ((fms >> 16) & 0xf) << 4; ecx_flags = ecx; edx_flags = edx; + cpuid_has_hv = ecx_flags & (1 << 31); if (!no_msr) { if (get_msr(sched_getcpu(), MSR_IA32_UCODE_REV, &ucode_patch)) @@ -8826,18 +9143,22 @@ void process_cpuid() fputc('\n', outf); fprintf(outf, "CPUID(0x80000000): max_extended_levels: 0x%x\n", max_extended_level); - fprintf(outf, "CPUID(1): %s %s %s %s %s %s %s %s %s %s\n", - ecx_flags & (1 << 0) ? "SSE3" : "-", - ecx_flags & (1 << 3) ? "MONITOR" : "-", - ecx_flags & (1 << 6) ? "SMX" : "-", - ecx_flags & (1 << 7) ? "EIST" : "-", - ecx_flags & (1 << 8) ? "TM2" : "-", - edx_flags & (1 << 4) ? "TSC" : "-", - edx_flags & (1 << 5) ? "MSR" : "-", - edx_flags & (1 << 22) ? "ACPI-TM" : "-", edx_flags & (1 << 28) ? "HT" : "-", edx_flags & (1 << 29) ? "TM" : "-"); + fprintf(outf, "CPUID(1): %sSSE3 %sMONITOR %sSMX %sEIST %sTM2 %sHV %sTSC %sMSR %sACPI-TM %sHT %sTM\n", + ecx_flags & (1 << 0) ? "" : "No-", + ecx_flags & (1 << 3) ? "" : "No-", + ecx_flags & (1 << 6) ? "" : "No-", + ecx_flags & (1 << 7) ? "" : "No-", + ecx_flags & (1 << 8) ? "" : "No-", + cpuid_has_hv ? "" : "No-", + edx_flags & (1 << 4) ? "" : "No-", + edx_flags & (1 << 5) ? "" : "No-", + edx_flags & (1 << 22) ? "" : "No-", edx_flags & (1 << 28) ? "" : "No-", edx_flags & (1 << 29) ? "" : "No-"); } + if (!quiet && cpuid_has_hv) + dump_cpuid_hypervisor(); probe_platform_features(family, model); + init_perf_model_support(family, model); if (!(edx_flags & (1 << 5))) errx(1, "CPUID: no MSR"); @@ -8887,7 +9208,7 @@ void process_cpuid() if (!quiet) decode_misc_enable_msr(); - if (max_level >= 0x7 && !quiet) { + if (max_level >= 0x7) { int has_sgx; ecx = 0; @@ -8896,9 +9217,10 @@ void process_cpuid() has_sgx = ebx & (1 << 2); - is_hybrid = edx & (1 << 15); + is_hybrid = !!(edx & (1 << 15)); - fprintf(outf, "CPUID(7): %sSGX %sHybrid\n", has_sgx ? "" : "No-", is_hybrid ? "" : "No-"); + if (!quiet) + fprintf(outf, "CPUID(7): %sSGX %sHybrid\n", has_sgx ? "" : "No-", is_hybrid ? "" : "No-"); if (has_sgx) decode_feature_control_msr(); @@ -8924,8 +9246,7 @@ void process_cpuid() if (crystal_hz) { tsc_hz = (unsigned long long)crystal_hz *ebx_tsc / eax_crystal; if (!quiet) - fprintf(outf, "TSC: %lld MHz (%d Hz * %d / %d / 1000000)\n", - tsc_hz / 1000000, crystal_hz, ebx_tsc, eax_crystal); + fprintf(outf, "TSC: %lld MHz (%d Hz * %d / %d / 1000000)\n", tsc_hz / 1000000, crystal_hz, ebx_tsc, eax_crystal); } } } @@ -9003,7 +9324,8 @@ void probe_pm_features(void) decode_misc_feature_control(); } -/* perf_llc_probe +/* + * has_perf_llc_access() * * return 1 on success, else 0 */ @@ -9014,7 +9336,7 @@ int has_perf_llc_access(void) if (no_perf) return 0; - fd = open_perf_counter(base_cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_REFERENCES, -1, PERF_FORMAT_GROUP); + fd = open_perf_counter(master_cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_REFERENCES, -1, PERF_FORMAT_GROUP); if (fd != -1) close(fd); @@ -9032,22 +9354,22 @@ void perf_llc_init(void) if (no_perf) return; - if (!(BIC_IS_ENABLED(BIC_LLC_RPS) && BIC_IS_ENABLED(BIC_LLC_HIT))) + if (!(BIC_IS_ENABLED(BIC_LLC_MRPS) || BIC_IS_ENABLED(BIC_LLC_HIT))) return; + assert(fd_llc_percpu != 0); + for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu) { if (cpu_is_not_allowed(cpu)) continue; - assert(fd_llc_percpu != 0); fd_llc_percpu[cpu] = open_perf_counter(cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_REFERENCES, -1, PERF_FORMAT_GROUP); if (fd_llc_percpu[cpu] == -1) { warnx("%s: perf REFS: failed to open counter on cpu%d", __func__, cpu); free_fd_llc_percpu(); return; } - assert(fd_llc_percpu != 0); retval = open_perf_counter(cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_MISSES, fd_llc_percpu[cpu], PERF_FORMAT_GROUP); if (retval == -1) { warnx("%s: perf MISS: failed to open counter on cpu%d", __func__, cpu); @@ -9055,10 +9377,90 @@ void perf_llc_init(void) return; } } - BIC_PRESENT(BIC_LLC_RPS); + BIC_PRESENT(BIC_LLC_MRPS); BIC_PRESENT(BIC_LLC_HIT); } +void perf_l2_init(void) +{ + int cpu; + int retval; + + if (no_perf) + return; + if (!(BIC_IS_ENABLED(BIC_L2_MRPS) || BIC_IS_ENABLED(BIC_L2_HIT))) + return; + if (perf_model_support == NULL) + return; + + assert(fd_l2_percpu != 0); + + for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu) { + + if (cpu_is_not_allowed(cpu)) + continue; + + if (!is_hybrid) { + fd_l2_percpu[cpu] = open_perf_counter(cpu, perf_pmu_types.uniform, perf_model_support->first.refs, -1, PERF_FORMAT_GROUP); + if (fd_l2_percpu[cpu] == -1) { + err(-1, "%s(cpu%d, 0x%x, 0x%llx) REFS", __func__, cpu, perf_pmu_types.uniform, perf_model_support->first.refs); + free_fd_l2_percpu(); + return; + } + retval = open_perf_counter(cpu, perf_pmu_types.uniform, perf_model_support->first.hits, fd_l2_percpu[cpu], PERF_FORMAT_GROUP); + if (retval == -1) { + err(-1, "%s(cpu%d, 0x%x, 0x%llx) HITS", __func__, cpu, perf_pmu_types.uniform, perf_model_support->first.hits); + free_fd_l2_percpu(); + return; + } + continue; + } + if (perf_pcore_set && CPU_ISSET_S(cpu, cpu_possible_setsize, perf_pcore_set)) { + fd_l2_percpu[cpu] = open_perf_counter(cpu, perf_pmu_types.pcore, perf_model_support->first.refs, -1, PERF_FORMAT_GROUP); + if (fd_l2_percpu[cpu] == -1) { + err(-1, "%s(cpu%d, 0x%x, 0x%llx) REFS", __func__, cpu, perf_pmu_types.pcore, perf_model_support->first.refs); + free_fd_l2_percpu(); + return; + } + retval = open_perf_counter(cpu, perf_pmu_types.pcore, perf_model_support->first.hits, fd_l2_percpu[cpu], PERF_FORMAT_GROUP); + if (retval == -1) { + err(-1, "%s(cpu%d, 0x%x, 0x%llx) HITS", __func__, cpu, perf_pmu_types.pcore, perf_model_support->first.hits); + free_fd_l2_percpu(); + return; + } + } else if (perf_ecore_set && CPU_ISSET_S(cpu, cpu_possible_setsize, perf_ecore_set)) { + fd_l2_percpu[cpu] = open_perf_counter(cpu, perf_pmu_types.ecore, perf_model_support->second.refs, -1, PERF_FORMAT_GROUP); + if (fd_l2_percpu[cpu] == -1) { + err(-1, "%s(cpu%d, 0x%x, 0x%llx) REFS", __func__, cpu, perf_pmu_types.pcore, perf_model_support->second.refs); + free_fd_l2_percpu(); + return; + } + retval = open_perf_counter(cpu, perf_pmu_types.ecore, perf_model_support->second.hits, fd_l2_percpu[cpu], PERF_FORMAT_GROUP); + if (retval == -1) { + err(-1, "%s(cpu%d, 0x%x, 0x%llx) HITS", __func__, cpu, perf_pmu_types.pcore, perf_model_support->second.hits); + free_fd_l2_percpu(); + return; + } + } else if (perf_lcore_set && CPU_ISSET_S(cpu, cpu_possible_setsize, perf_lcore_set)) { + fd_l2_percpu[cpu] = open_perf_counter(cpu, perf_pmu_types.lcore, perf_model_support->third.refs, -1, PERF_FORMAT_GROUP); + if (fd_l2_percpu[cpu] == -1) { + err(-1, "%s(cpu%d, 0x%x, 0x%llx) REFS", __func__, cpu, perf_pmu_types.pcore, perf_model_support->third.refs); + free_fd_l2_percpu(); + return; + } + retval = open_perf_counter(cpu, perf_pmu_types.lcore, perf_model_support->third.hits, fd_l2_percpu[cpu], PERF_FORMAT_GROUP); + if (retval == -1) { + err(-1, "%s(cpu%d, 0x%x, 0x%llx) HITS", __func__, cpu, perf_pmu_types.pcore, perf_model_support->third.hits); + free_fd_l2_percpu(); + return; + } + } else + err(-1, "%s: cpu%d: type %d", __func__, cpu, cpus[cpu].type); + } + BIC_PRESENT(BIC_L2_MRPS); + BIC_PRESENT(BIC_L2_HIT); +} + /* * in /dev/cpu/ return success for names that are numbers * ie. filter out ".", "..", "microcode". @@ -9071,33 +9473,6 @@ int dir_filter(const struct dirent *dirp) return 0; } -char *possible_file = "/sys/devices/system/cpu/possible"; -char possible_buf[1024]; - -int initialize_cpu_possible_set(void) -{ - FILE *fp; - - fp = fopen(possible_file, "r"); - if (!fp) { - warn("open %s", possible_file); - return -1; - } - if (fread(possible_buf, sizeof(char), 1024, fp) == 0) { - warn("read %s", possible_file); - goto err; - } - if (parse_cpu_str(possible_buf, cpu_possible_set, cpu_possible_setsize)) { - warnx("%s: cpu str malformat %s\n", possible_file, cpu_effective_str); - goto err; - } - return 0; - -err: - fclose(fp); - return -1; -} - void topology_probe(bool startup) { int i; @@ -9137,7 +9512,7 @@ void topology_probe(bool startup) err(3, "CPU_ALLOC"); cpu_possible_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1)); CPU_ZERO_S(cpu_possible_setsize, cpu_possible_set); - initialize_cpu_possible_set(); + initialize_cpu_set_from_sysfs(cpu_possible_set, "/sys/devices/system/cpu", "possible"); /* * Allocate and initialize cpu_effective_set @@ -9205,13 +9580,13 @@ void topology_probe(bool startup) cpu_affinity_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1)); CPU_ZERO_S(cpu_affinity_setsize, cpu_affinity_set); - for_all_proc_cpus(init_thread_id); + for_all_proc_cpus(clear_ht_id); for_all_proc_cpus(set_cpu_hybrid_type); /* * For online cpus - * find max_core_id, max_package_id + * find max_core_id, max_package_id, num_cores (per system) */ for (i = 0; i <= topo.max_cpu_num; ++i) { int siblings; @@ -9222,12 +9597,12 @@ void topology_probe(bool startup) continue; } - cpus[i].logical_cpu_id = i; + cpus[i].cpu_id = i; /* get package information */ - cpus[i].physical_package_id = get_physical_package_id(i); - if (cpus[i].physical_package_id > max_package_id) - max_package_id = cpus[i].physical_package_id; + cpus[i].package_id = get_package_id(i); + if (cpus[i].package_id > max_package_id) + max_package_id = cpus[i].package_id; /* get die information */ cpus[i].die_id = get_die_id(i); @@ -9245,18 +9620,18 @@ void topology_probe(bool startup) topo.max_node_num = cpus[i].physical_node_id; /* get core information */ - cpus[i].physical_core_id = get_core_id(i); - if (cpus[i].physical_core_id > max_core_id) - max_core_id = cpus[i].physical_core_id; + cpus[i].core_id = get_core_id(i); + if (cpus[i].core_id > max_core_id) + max_core_id = cpus[i].core_id; /* get thread information */ - siblings = get_thread_siblings(&cpus[i]); + siblings = set_thread_siblings(&cpus[i]); if (siblings > max_siblings) max_siblings = siblings; - if (cpus[i].thread_id == 0) + if (cpus[i].ht_id == 0) topo.num_cores++; } - topo.max_core_id = max_core_id; + topo.max_core_id = max_core_id; /* within a package */ topo.max_package_id = max_package_id; topo.cores_per_node = max_core_id + 1; @@ -9298,42 +9673,57 @@ void topology_probe(bool startup) continue; fprintf(outf, "cpu %d pkg %d die %d l3 %d node %d lnode %d core %d thread %d\n", - i, cpus[i].physical_package_id, cpus[i].die_id, cpus[i].l3_id, - cpus[i].physical_node_id, cpus[i].logical_node_id, cpus[i].physical_core_id, cpus[i].thread_id); + i, cpus[i].package_id, cpus[i].die_id, cpus[i].l3_id, + cpus[i].physical_node_id, cpus[i].logical_node_id, cpus[i].core_id, cpus[i].ht_id); } } -void allocate_counters(struct thread_data **t, struct core_data **c, struct pkg_data **p) +void allocate_counters_1(struct counters *counters) +{ + counters->threads = calloc(1, sizeof(struct thread_data)); + if (counters->threads == NULL) + goto error; + + counters->cores = calloc(1, sizeof(struct core_data)); + if (counters->cores == NULL) + goto error; + + counters->packages = calloc(1, sizeof(struct pkg_data)); + if (counters->packages == NULL) + goto error; + + return; +error: + err(1, "calloc counters_1"); +} + +void allocate_counters(struct counters *counters) { int i; int num_cores = topo.cores_per_node * topo.nodes_per_pkg * topo.num_packages; int num_threads = topo.threads_per_core * num_cores; - *t = calloc(num_threads, sizeof(struct thread_data)); - if (*t == NULL) + counters->threads = calloc(num_threads, sizeof(struct thread_data)); + if (counters->threads == NULL) goto error; for (i = 0; i < num_threads; i++) - (*t)[i].cpu_id = -1; + (counters->threads)[i].cpu_id = -1; - *c = calloc(num_cores, sizeof(struct core_data)); - if (*c == NULL) + counters->cores = calloc(num_cores, sizeof(struct core_data)); + if (counters->cores == NULL) goto error; - for (i = 0; i < num_cores; i++) { - (*c)[i].core_id = -1; - (*c)[i].base_cpu = -1; - } + for (i = 0; i < num_cores; i++) + (counters->cores)[i].first_cpu = -1; - *p = calloc(topo.num_packages, sizeof(struct pkg_data)); - if (*p == NULL) + counters->packages = calloc(topo.num_packages, sizeof(struct pkg_data)); + if (counters->packages == NULL) goto error; - for (i = 0; i < topo.num_packages; i++) { - (*p)[i].package_id = i; - (*p)[i].base_cpu = -1; - } + for (i = 0; i < topo.num_packages; i++) + (counters->packages)[i].first_cpu = -1; return; error: @@ -9343,14 +9733,13 @@ error: /* * init_counter() * - * set FIRST_THREAD_IN_CORE and FIRST_CORE_IN_PACKAGE + * set t->cpu_id, FIRST_THREAD_IN_CORE and FIRST_CORE_IN_PACKAGE */ void init_counter(struct thread_data *thread_base, struct core_data *core_base, struct pkg_data *pkg_base, int cpu_id) { - int pkg_id = cpus[cpu_id].physical_package_id; + int pkg_id = cpus[cpu_id].package_id; int node_id = cpus[cpu_id].logical_node_id; - int core_id = cpus[cpu_id].physical_core_id; - int thread_id = cpus[cpu_id].thread_id; + int core_id = cpus[cpu_id].core_id; struct thread_data *t; struct core_data *c; @@ -9360,20 +9749,17 @@ void init_counter(struct thread_data *thread_base, struct core_data *core_base, if (node_id < 0) node_id = 0; - t = GET_THREAD(thread_base, thread_id, core_id, node_id, pkg_id); - c = GET_CORE(core_base, core_id, node_id, pkg_id); + t = &thread_base[cpu_id]; + c = &core_base[GLOBAL_CORE_ID(core_id, pkg_id)]; t->cpu_id = cpu_id; if (!cpu_is_not_allowed(cpu_id)) { - if (c->base_cpu < 0) - c->base_cpu = t->cpu_id; - if (pkg_base[pkg_id].base_cpu < 0) - pkg_base[pkg_id].base_cpu = t->cpu_id; + if (c->first_cpu < 0) + c->first_cpu = t->cpu_id; + if (pkg_base[pkg_id].first_cpu < 0) + pkg_base[pkg_id].first_cpu = t->cpu_id; } - - c->core_id = core_id; - pkg_base[pkg_id].package_id = pkg_id; } int initialize_counters(int cpu_id) @@ -9416,9 +9802,9 @@ void allocate_irq_buffers(void) int update_topo(PER_THREAD_PARAMS) { topo.allowed_cpus++; - if ((int)t->cpu_id == c->base_cpu) + if ((int)t->cpu_id == c->first_cpu) topo.allowed_cores++; - if ((int)t->cpu_id == p->base_cpu) + if ((int)t->cpu_id == p->first_cpu) topo.allowed_packages++; return 0; @@ -9437,23 +9823,24 @@ void setup_all_buffers(bool startup) topology_probe(startup); allocate_irq_buffers(); allocate_fd_percpu(); - allocate_counters(&thread_even, &core_even, &package_even); - allocate_counters(&thread_odd, &core_odd, &package_odd); + allocate_counters_1(&average); + allocate_counters(&even); + allocate_counters(&odd); allocate_output_buffer(); for_all_proc_cpus(initialize_counters); topology_update(); } -void set_base_cpu(void) +void set_master_cpu(void) { int i; for (i = 0; i < topo.max_cpu_num + 1; ++i) { if (cpu_is_not_allowed(i)) continue; - base_cpu = i; + master_cpu = i; if (debug > 1) - fprintf(outf, "base_cpu = %d\n", base_cpu); + fprintf(outf, "master_cpu = %d\n", master_cpu); return; } err(-ENODEV, "No valid cpus found"); @@ -9484,7 +9871,7 @@ void check_perf_access(void) if (!has_perf_instr_count_access()) no_perf = 1; - if (BIC_IS_ENABLED(BIC_LLC_RPS) || BIC_IS_ENABLED(BIC_LLC_HIT)) + if (BIC_IS_ENABLED(BIC_LLC_MRPS) || BIC_IS_ENABLED(BIC_LLC_HIT)) if (!has_perf_llc_access()) no_perf = 1; @@ -9967,8 +10354,7 @@ void pmt_init(void) if (BIC_IS_ENABLED(BIC_Diec6)) { pmt_add_counter(PMT_MTL_DC6_GUID, PMT_MTL_DC6_SEQ, "Die%c6", PMT_TYPE_XTAL_TIME, - PMT_COUNTER_MTL_DC6_LSB, PMT_COUNTER_MTL_DC6_MSB, PMT_COUNTER_MTL_DC6_OFFSET, - SCOPE_PACKAGE, FORMAT_DELTA, 0, PMT_OPEN_TRY); + PMT_COUNTER_MTL_DC6_LSB, PMT_COUNTER_MTL_DC6_MSB, PMT_COUNTER_MTL_DC6_OFFSET, SCOPE_PACKAGE, FORMAT_DELTA, 0, PMT_OPEN_TRY); } if (BIC_IS_ENABLED(BIC_CPU_c1e)) { @@ -10029,7 +10415,7 @@ next_loop_iter: void turbostat_init() { setup_all_buffers(true); - set_base_cpu(); + set_master_cpu(); check_msr_access(); check_perf_access(); process_cpuid(); @@ -10040,13 +10426,14 @@ void turbostat_init() rapl_perf_init(); cstate_perf_init(); perf_llc_init(); + perf_l2_init(); added_perf_counters_init(); pmt_init(); for_all_cpus(get_cpu_type, ODD_COUNTERS); for_all_cpus(get_cpu_type, EVEN_COUNTERS); - if (BIC_IS_ENABLED(BIC_IPC) && has_aperf_access && get_instr_count_fd(base_cpu) != -1) + if (BIC_IS_ENABLED(BIC_IPC) && has_aperf_access && get_instr_count_fd(master_cpu) != -1) BIC_PRESENT(BIC_IPC); /* @@ -10145,7 +10532,7 @@ int get_and_dump_counters(void) void print_version() { - fprintf(outf, "turbostat version 2025.12.02 - Len Brown \n"); + fprintf(outf, "turbostat version 2026.02.14 - Len Brown \n"); } #define COMMAND_LINE_SIZE 2048 @@ -10767,8 +11154,7 @@ next: } if (direct_path && has_guid) { - printf("%s: path and guid+seq parameters are mutually exclusive\n" - "notice: passed guid=0x%x and path=%s\n", __func__, guid, direct_path); + printf("%s: path and guid+seq parameters are mutually exclusive\nnotice: passed guid=0x%x and path=%s\n", __func__, guid, direct_path); exit(1); } @@ -10863,7 +11249,7 @@ void probe_cpuidle_residency(void) for (state = 10; state >= 0; --state) { - sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name", base_cpu, state); + sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name", master_cpu, state); input = fopen(path, "r"); if (input == NULL) continue; @@ -10912,7 +11298,7 @@ void probe_cpuidle_counts(void) for (state = 10; state >= 0; --state) { - sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name", base_cpu, state); + sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name", master_cpu, state); input = fopen(path, "r"); if (input == NULL) continue; @@ -11041,7 +11427,7 @@ void cmdline(int argc, char **argv) * Parse some options early, because they may make other options invalid, * like adding the MSR counter with --add and at the same time using --no-msr. */ - while ((opt = getopt_long_only(argc, argv, "+MPn:", long_options, &option_index)) != -1) { + while ((opt = getopt_long_only(argc, argv, "+:MP", long_options, &option_index)) != -1) { switch (opt) { case 'M': no_msr = 1; @@ -11055,7 +11441,7 @@ void cmdline(int argc, char **argv) } optind = 0; - while ((opt = getopt_long_only(argc, argv, "+C:c:Dde:hi:Jn:o:qMST:v", long_options, &option_index)) != -1) { + while ((opt = getopt_long_only(argc, argv, "+C:c:Dde:hi:Jn:N:o:qMST:v", long_options, &option_index)) != -1) { switch (opt) { case 'a': parse_add_command(optarg); @@ -11098,7 +11484,6 @@ void cmdline(int argc, char **argv) } break; case 'h': - default: help(); exit(1); case 'i': @@ -11134,20 +11519,18 @@ void cmdline(int argc, char **argv) /* Parsed earlier */ break; case 'n': - num_iterations = strtod(optarg, NULL); + num_iterations = strtoul(optarg, NULL, 0); + errno = 0; - if (num_iterations <= 0) { - fprintf(outf, "iterations %d should be positive number\n", num_iterations); - exit(2); - } + if (errno || num_iterations == 0) + errx(-1, "invalid iteration count: %s", optarg); break; case 'N': - header_iterations = strtod(optarg, NULL); + header_iterations = strtoul(optarg, NULL, 0); + errno = 0; - if (header_iterations <= 0) { - fprintf(outf, "iterations %d should be positive number\n", header_iterations); - exit(2); - } + if (errno || header_iterations == 0) + errx(-1, "invalid header iteration count: %s", optarg); break; case 's': /* @@ -11170,6 +11553,9 @@ void cmdline(int argc, char **argv) print_version(); exit(0); break; + default: + help(); + exit(1); } } }